mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(vertex_ai): stop a json_schema row failing the plain rows in its batch file
Vertex loads a batch file into one BigQuery table, so each row's generationConfig gains a null for every key another row sets. response_json_schema is a google.protobuf.Value, so its null counts as set and every plain row with a generationConfig of its own fails with "response_mime_type must be set". Proto JSON takes both spellings of the field, so a row carrying a schema now uses generation_config, which no other row shares. The request Vertex parses is the same as before.
This commit is contained in:
parent
02ef44c388
commit
97b10c9151
2 changed files with 70 additions and 1 deletions
|
|
@ -826,6 +826,20 @@ def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
|
|||
return {"custom_id": str(custom_id), "request": request}
|
||||
|
||||
|
||||
def _with_json_schema_config_in_its_own_column(vertex_request_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""
|
||||
Vertex loads a batch file into one BigQuery table, so each row's `generationConfig` gains a null for every key
|
||||
another row sets. `response_json_schema` is a `google.protobuf.Value`, so its null counts as set and fails the
|
||||
row. Proto JSON takes both spellings of the field, so a row with a schema uses the one no other row shares.
|
||||
"""
|
||||
generation_config: Final = vertex_request_body.get("generationConfig")
|
||||
if not isinstance(generation_config, Mapping) or "response_json_schema" not in generation_config:
|
||||
return vertex_request_body
|
||||
return {
|
||||
("generation_config" if key == "generationConfig" else key): value for key, value in vertex_request_body.items()
|
||||
}
|
||||
|
||||
|
||||
def _openai_batch_jsonl_entry_to_vertex_rows(
|
||||
openai_entry: dict[str, Any],
|
||||
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
|
||||
|
|
@ -889,7 +903,7 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
|
|||
vertex_request_body["labels"] = {}
|
||||
_set_litellm_batch_custom_id_labels(vertex_request_body["labels"], custom_id)
|
||||
|
||||
return ({"request": vertex_request_body},)
|
||||
return ({"request": _with_json_schema_config_in_its_own_column(vertex_request_body)},)
|
||||
|
||||
|
||||
def _decode_batch_input_line(lineno: int, raw: str | bytes | bytearray) -> str:
|
||||
|
|
|
|||
|
|
@ -2474,3 +2474,58 @@ def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: py
|
|||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)])
|
||||
assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS}
|
||||
|
||||
|
||||
_ANSWER_SCHEMA: Final = MappingProxyType(
|
||||
{"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}
|
||||
)
|
||||
_JSON_SCHEMA_KNOBS: Final = MappingProxyType(
|
||||
{"response_format": {"type": "json_schema", "json_schema": {"name": "answer", "schema": dict(_ANSWER_SCHEMA)}}}
|
||||
)
|
||||
|
||||
|
||||
def _request_of(row: Mapping[str, object]) -> dict[str, object]:
|
||||
return TypeAdapter(dict[str, object]).validate_python(row["request"])
|
||||
|
||||
|
||||
def _config_columns(row: Mapping[str, object]) -> frozenset[str]:
|
||||
return frozenset(_request_of(row)) & {"generationConfig", "generation_config"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"plain_knobs",
|
||||
[{}, {"temperature": 0.2}, {"response_format": {"type": "json_object"}}, {"response_format": {"type": "text"}}],
|
||||
ids=["max_tokens", "temperature", "json_object", "text"],
|
||||
)
|
||||
def test_a_plain_row_shares_no_config_column_with_a_json_schema_row(plain_knobs: Mapping[str, object]) -> None:
|
||||
"""Vertex loads a batch file as one table, so a shared column gives the plain row a null response_json_schema,
|
||||
which fails it with 'response_mime_type must be set when response_json_schema is set'"""
|
||||
(plain_row, schema_row) = _wrap_entries(
|
||||
[_chat_entry("gemini-2.5-flash", plain_knobs), _chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS)]
|
||||
)
|
||||
assert _config_columns(plain_row).isdisjoint(_config_columns(schema_row))
|
||||
|
||||
|
||||
def test_a_json_schema_row_keeps_its_whole_generation_config() -> None:
|
||||
(row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS)])
|
||||
assert _request_of(row)["generation_config"] == {
|
||||
"max_output_tokens": 5,
|
||||
"response_mime_type": "application/json",
|
||||
"response_json_schema": dict(_ANSWER_SCHEMA),
|
||||
}
|
||||
|
||||
|
||||
def test_a_responses_row_with_a_json_schema_uses_the_same_column_as_a_chat_row() -> None:
|
||||
(responses_row, chat_row) = _wrap_entries(
|
||||
[
|
||||
_responses_entry(
|
||||
{
|
||||
"model": "gemini-2.5-flash",
|
||||
"input": "hi",
|
||||
"text": {"format": {"type": "json_schema", "name": "answer", "schema": dict(_ANSWER_SCHEMA)}},
|
||||
}
|
||||
),
|
||||
_chat_entry("gemini-2.5-flash", _JSON_SCHEMA_KNOBS),
|
||||
]
|
||||
)
|
||||
assert _config_columns(responses_row) == _config_columns(chat_row) == {"generation_config"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue