diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 9159c9a4070..5b465b166f5 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -826,6 +826,20 @@ def _openai_batch_jsonl_entry_to_vertex_anthropic_row( return {"custom_id": str(custom_id), "request": request} +def _with_json_schema_config_in_its_own_column(vertex_request_body: Mapping[str, object]) -> Mapping[str, object]: + """ + Vertex loads a batch file into one BigQuery table, so each row's `generationConfig` gains a null for every key + another row sets. `response_json_schema` is a `google.protobuf.Value`, so its null counts as set and fails the + row. Proto JSON takes both spellings of the field, so a row with a schema uses the one no other row shares. + """ + generation_config: Final = vertex_request_body.get("generationConfig") + if not isinstance(generation_config, Mapping) or "response_json_schema" not in generation_config: + return vertex_request_body + return { + ("generation_config" if key == "generationConfig" else key): value for key, value in vertex_request_body.items() + } + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], @@ -889,7 +903,7 @@ def _openai_batch_jsonl_entry_to_vertex_rows( vertex_request_body["labels"] = {} _set_litellm_batch_custom_id_labels(vertex_request_body["labels"], custom_id) - return ({"request": vertex_request_body},) + return ({"request": _with_json_schema_config_in_its_own_column(vertex_request_body)},) def _decode_batch_input_line(lineno: int, raw: str | bytes | bytearray) -> str: diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 98e204dd44e..be818daa274 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -2474,3 +2474,58 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py monkeypatch.setattr(litellm, "drop_params", False) (row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)]) assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS} + + +_ANSWER_SCHEMA: Final = MappingProxyType( + {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]} +) +_JSON_SCHEMA_KNOBS: Final = MappingProxyType( + {"response_format": {"type": "json_schema", "json_schema": {"name": "answer", "schema": dict(_ANSWER_SCHEMA)}}} +) + + +def _request_of(row: Mapping[str, object]) -> dict[str, object]: + return TypeAdapter(dict[str, object]).validate_python(row["request"]) + + +def _config_columns(row: Mapping[str, object]) -> frozenset[str]: + return frozenset(_request_of(row)) & {"generationConfig", "generation_config"} + + +@pytest.mark.parametrize( + "plain_knobs", + [{}, {"temperature": 0.2}, {"response_format": {"type": "json_object"}}, {"response_format": {"type": "text"}}], + ids=["max_tokens", "temperature", "json_object", "text"], +) +def test_a_plain_row_shares_no_config_column_with_a_json_schema_row(plain_knobs: Mapping[str, object]) -> None: + """Vertex loads a batch file as one table, so a shared column gives the plain row a null response_json_schema, + which fails it with 'response_mime_type must be set when response_json_schema is set'""" + (plain_row, schema_row) = _wrap_entries( + [_chat_entry("gemini-2.5-flash", plain_knobs), _chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS)] + ) + assert _config_columns(plain_row).isdisjoint(_config_columns(schema_row)) + + +def test_a_json_schema_row_keeps_its_whole_generation_config() -> None: + (row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS)]) + assert _request_of(row)["generation_config"] == { + "max_output_tokens": 5, + "response_mime_type": "application/json", + "response_json_schema": dict(_ANSWER_SCHEMA), + } + + +def test_a_responses_row_with_a_json_schema_uses_the_same_column_as_a_chat_row() -> None: + (responses_row, chat_row) = _wrap_entries( + [ + _responses_entry( + { + "model": "gemini-2.5-flash", + "input": "hi", + "text": {"format": {"type": "json_schema", "name": "answer", "schema": dict(_ANSWER_SCHEMA)}}, + } + ), + _chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS), + ] + ) + assert _config_columns(responses_row) == _config_columns(chat_row) == {"generation_config"}