fix(vertex_ai): stop a json_schema row failing the plain rows in its batch file

Vertex loads a batch file into one BigQuery table, so each row's generationConfig
gains a null for every key another row sets. response_json_schema is a
google.protobuf.Value, so its null counts as set and every plain row with a
generationConfig of its own fails with "response_mime_type must be set".

Proto JSON takes both spellings of the field, so a row carrying a schema now uses
generation_config, which no other row shares. The request Vertex parses is the
same as before.
This commit is contained in:
mateo-berri 2026-10-30 06:25:47 -07:00
parent 02ef44c388
commit 97b10c9151
2 changed files with 70 additions and 1 deletions

View file

@ -826,6 +826,20 @@ def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
return {"custom_id": str(custom_id), "request": request}
def _with_json_schema_config_in_its_own_column(vertex_request_body: Mapping[str, object]) -> Mapping[str, object]:
"""
Vertex loads a batch file into one BigQuery table, so each row's `generationConfig` gains a null for every key
another row sets. `response_json_schema` is a `google.protobuf.Value`, so its null counts as set and fails the
row. Proto JSON takes both spellings of the field, so a row with a schema uses the one no other row shares.
"""
generation_config: Final = vertex_request_body.get("generationConfig")
if not isinstance(generation_config, Mapping) or "response_json_schema" not in generation_config:
return vertex_request_body
return {
("generation_config" if key == "generationConfig" else key): value for key, value in vertex_request_body.items()
}
def _openai_batch_jsonl_entry_to_vertex_rows(
openai_entry: dict[str, Any],
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
@ -889,7 +903,7 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
vertex_request_body["labels"] = {}
_set_litellm_batch_custom_id_labels(vertex_request_body["labels"], custom_id)
return ({"request": vertex_request_body},)
return ({"request": _with_json_schema_config_in_its_own_column(vertex_request_body)},)
def _decode_batch_input_line(lineno: int, raw: str | bytes | bytearray) -> str:

View file

@ -2474,3 +2474,58 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py
monkeypatch.setattr(litellm, "drop_params", False)
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)])
assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS}
_ANSWER_SCHEMA: Final = MappingProxyType(
{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}
)
_JSON_SCHEMA_KNOBS: Final = MappingProxyType(
{"response_format": {"type": "json_schema", "json_schema": {"name": "answer", "schema": dict(_ANSWER_SCHEMA)}}}
)
def _request_of(row: Mapping[str, object]) -> dict[str, object]:
return TypeAdapter(dict[str, object]).validate_python(row["request"])
def _config_columns(row: Mapping[str, object]) -> frozenset[str]:
return frozenset(_request_of(row)) & {"generationConfig", "generation_config"}
@pytest.mark.parametrize(
"plain_knobs",
[{}, {"temperature": 0.2}, {"response_format": {"type": "json_object"}}, {"response_format": {"type": "text"}}],
ids=["max_tokens", "temperature", "json_object", "text"],
)
def test_a_plain_row_shares_no_config_column_with_a_json_schema_row(plain_knobs: Mapping[str, object]) -> None:
"""Vertex loads a batch file as one table, so a shared column gives the plain row a null response_json_schema,
which fails it with 'response_mime_type must be set when response_json_schema is set'"""
(plain_row, schema_row) = _wrap_entries(
[_chat_entry("gemini-2.5-flash", plain_knobs), _chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS)]
)
assert _config_columns(plain_row).isdisjoint(_config_columns(schema_row))
def test_a_json_schema_row_keeps_its_whole_generation_config() -> None:
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS)])
assert _request_of(row)["generation_config"] == {
"max_output_tokens": 5,
"response_mime_type": "application/json",
"response_json_schema": dict(_ANSWER_SCHEMA),
}
def test_a_responses_row_with_a_json_schema_uses_the_same_column_as_a_chat_row() -> None:
(responses_row, chat_row) = _wrap_entries(
[
_responses_entry(
{
"model": "gemini-2.5-flash",
"input": "hi",
"text": {"format": {"type": "json_schema", "name": "answer", "schema": dict(_ANSWER_SCHEMA)}},
}
),
_chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS),
]
)
assert _config_columns(responses_row) == _config_columns(chat_row) == {"generation_config"}