fix(vertex_ai): build Claude batch rows with chat's param mapping and surface Vertex batch create errors

Claude batch rows now go through get_optional_params like Vertex Claude chat, so unsupported params, top_k, response_format and extra_body follow drop_params. Tool names Claude would rename get a 400 at upload. A refused batch create returns Vertex's own status and message.
This commit is contained in:
mateo-berri 2026-10-28 20:20:41 -07:00
parent 3fa19ced6d
commit 0879b99b99
5 changed files with 222 additions and 65 deletions

View file

@ -81,6 +81,13 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str
return response.json()
def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> VertexAIError:
return VertexAIError(
status_code=error.response.status_code,
message=f"Vertex AI batch create failed: {error.response.text}",
)
class VertexAIBatchPrediction(VertexLLM):
def __init__(self, gcs_bucket_name: str, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -166,11 +173,14 @@ class VertexAIBatchPrediction(VertexLLM):
headers=headers,
)
response: Final = sync_handler.post(
url=api_base,
headers=headers,
data=json.dumps(vertex_batch_request),
)
try:
response: Final = sync_handler.post(
url=api_base,
headers=headers,
data=json.dumps(vertex_batch_request),
)
except httpx.HTTPStatusError as e:
raise _vertex_batch_create_error(e) from e
payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
_json_response: Final = payload_view["payload"]
@ -266,13 +276,7 @@ class VertexAIBatchPrediction(VertexLLM):
data=json.dumps(vertex_batch_request),
)
except httpx.HTTPStatusError as e:
error_body: Final = e.response.text
litellm.verbose_logger.error(
"Vertex AI batch create failed: status=%s, body=%s",
e.response.status_code,
error_body[:1000],
)
raise
raise _vertex_batch_create_error(e) from e
payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
_json_response: Final = payload_view["payload"]

View file

@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._uuid import uuid
from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES
from litellm.files.types import FileContentStreamingResult
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.cloud_storage_security import (
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_metadata,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
@ -86,6 +88,7 @@ from litellm.types.utils import (
ModelResponse,
Usage,
)
from litellm.utils import get_non_default_completion_params, get_optional_params
from ..common_utils import VertexAIError
from ..vertex_llm_base import VertexBase
@ -462,6 +465,7 @@ def _vertex_anthropic_batch_output_row_to_openai(
completion_response=dict(anthropic_response),
raw_response=mock_httpx_response,
model_response=ModelResponse(),
json_mode=True,
)
except Exception as e: # noqa: BLE001 # one malformed row must fail alone, not the whole output file
return _openai_batch_output_row(
@ -746,9 +750,14 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
)
_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset(("model", "messages", "custom_llm_provider", "drop_params"))
_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"}
def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
openai_entry: Mapping[str, object],
chat_request_body: Mapping[str, Any],
custom_id: object,
messages: list[AllMessageValues],
request_params: Mapping[str, object],
model: str,
deployment_drop_params: object,
) -> Mapping[str, object]:
@ -758,23 +767,51 @@ def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
"""
config: Final = VertexAIAnthropicConfig()
bare_model: Final = model.rsplit("/", 1)[-1]
optional_params: Final[Mapping[str, object]] = config.map_openai_params(
non_default_params={key: value for key, value in chat_request_body.items() if key not in ("model", "messages")},
optional_params={},
drop_params: Final = drops_unsupported_params(request_params.get("drop_params", deployment_drop_params))
chat_params: Final = {
**get_non_default_completion_params(request_params),
**{key: value for key, value in request_params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS},
}
optional_params: Final[Mapping[str, object]] = get_optional_params(
model=bare_model,
drop_params=drops_unsupported_params(chat_request_body.get("drop_params", deployment_drop_params)),
custom_llm_provider="vertex_ai",
drop_params=drop_params,
messages=messages,
**{ # pyright: ignore[reportArgumentType] # batch JSONL bodies are untyped; get_optional_params checks each param as chat does
key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS
},
)
request_state: Final[dict[str, object]] = {"drop_params": drop_params}
anthropic_request: Final[Mapping[str, object]] = config.transform_request(
model=bare_model,
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
optional_params={**optional_params, "anthropic_version": VERTEX_ANTHROPIC_VERSION},
litellm_params={},
messages=messages,
optional_params={
**{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")},
"anthropic_version": VERTEX_ANTHROPIC_VERSION,
},
litellm_params=request_state,
headers={},
)
custom_id: Final = openai_entry.get("custom_id")
renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY))
if renamed_tools:
raise ValueError(
f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, "
"and batch output cannot map a renamed tool back, so rename these tools"
)
extra_body: Final = as_json_object(optional_params.get("extra_body"))
request: Final[Mapping[str, object]] = (
anthropic_request
if extra_body is None
else {
**anthropic_request,
**config.transform_extra_body(
extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={}
),
}
)
if custom_id is None:
return {"request": anthropic_request}
return {"custom_id": str(custom_id), "request": anthropic_request}
return {"request": request}
return {"custom_id": str(custom_id), "request": request}
def _openai_batch_jsonl_entry_to_vertex_rows(
@ -812,14 +849,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
if _is_responses_batch_entry(openai_entry)
else openai_request_body
)
chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", []))
if is_anthropic_model:
return (
_openai_batch_jsonl_entry_to_vertex_anthropic_row(
openai_entry, chat_request_body, model, deployment_drop_params
custom_id=openai_entry.get("custom_id"),
messages=chat_messages,
request_params={
key: value for key, value in chat_request_body.items() if key not in ("model", "messages")
},
model=model,
deployment_drop_params=deployment_drop_params,
),
)
vertex_request_body: Final = transform_request_body(
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
messages=chat_messages,
model=chat_request_body.get("model", ""),
optional_params=map_openai_to_vertex_params(chat_request_body),
custom_llm_provider="vertex_ai",

View file

@ -370,9 +370,7 @@ def test_create_batch_sync_endpoint_without_deployed_model_raises_400():
client.post.assert_not_called()
def test_create_batch_sync_httpstatuserror_propagates():
"""``HTTPHandler.post`` raises for non-2xx via ``raise_for_status``; the
sync create path must surface that error, not swallow it."""
def test_create_batch_sync_http_error_surfaces_vertex_status_and_body():
h = _make_handler()
client = MagicMock()
request = httpx.Request("POST", "https://x/batchPredictionJobs")
@ -382,7 +380,7 @@ def test_create_batch_sync_httpstatuserror_propagates():
)
with patch(f"{HMOD}.get_httpx_client", return_value=client):
with pytest.raises(httpx.HTTPStatusError):
with pytest.raises(VertexAIError, match="boom") as exc_info:
h.create_batch(
_is_async=False,
create_batch_data=CREATE_DATA,
@ -393,6 +391,7 @@ def test_create_batch_sync_httpstatuserror_propagates():
timeout=600.0,
max_retries=None,
)
assert exc_info.value.status_code == 500
def test_create_batch_input_file_id_without_model_raises_400_before_post():
@ -800,7 +799,7 @@ def test_cancel_batch_sync_httpstatuserror_logged_and_reraised():
client.get.assert_not_called()
def test_create_batch_async_httpstatuserror_logged_and_reraised():
def test_create_batch_async_http_error_surfaces_vertex_status_and_body():
h = _make_handler()
async_client = MagicMock()
request = httpx.Request("POST", "https://x/batchPredictionJobs")
@ -825,8 +824,9 @@ def test_create_batch_async_httpstatuserror_logged_and_reraised():
timeout=600.0,
max_retries=None,
)
with pytest.raises(httpx.HTTPStatusError):
with pytest.raises(VertexAIError, match="boom") as exc_info:
_run(coro)
assert exc_info.value.status_code == 500
def test_async_retrieve_batch_non_200_raises():
@ -1043,15 +1043,7 @@ async def test_vertex_list_batches(monkeypatch):
@pytest.mark.asyncio
async def test_vertex_async_create_batch_logs_error_body_on_http_error():
"""
When Vertex AI returns an HTTP error (e.g. 400), _async_create_batch should
re-raise httpx.HTTPStatusError (not swallow it) and log the response body.
Before the fix the error body was lost because AsyncHTTPHandler.post()
calls raise_for_status() internally, raising before the handler's own
status-code check could log the body.
"""
async def test_vertex_async_create_batch_error_carries_vertex_error_body():
from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction
handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket")
@ -1073,12 +1065,12 @@ async def test_vertex_async_create_batch_logs_error_body_on_http_error():
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
side_effect=http_error,
):
with pytest.raises(httpx.HTTPStatusError) as exc_info:
with pytest.raises(VertexAIError) as exc_info:
await handler._async_create_batch(
vertex_batch_request={},
api_base="https://us-central1-aiplatform.googleapis.com/v1/projects/test/locations/us-central1/batchPredictionJobs",
headers={"Authorization": "Bearer fake-token"},
)
assert exc_info.value.response.status_code == 400
assert "gemini-2.0-flash" in exc_info.value.response.text
assert exc_info.value.status_code == 400
assert "Do not support publisher model gemini-2.0-flash" in exc_info.value.message

View file

@ -524,10 +524,9 @@ def test_is_vertex_anthropic_batch_model(model: str, expected: bool):
def test_claude_output_row_is_costed_from_anthropic_usage_and_its_own_model():
priced_models: Final[list[str]] = []
def cost_calculator(usage, model, custom_llm_provider=None, model_info=None, vertex_location=None) -> tuple[float, float]:
priced_models.append(model)
if model != "claude-sonnet-4-6":
return 0.0, 0.0
return usage.prompt_tokens * 0.001, usage.completion_tokens * 0.002
stats = native_vertex_batch_row_stats(
@ -552,5 +551,5 @@ def test_claude_output_row_is_costed_from_anthropic_usage_and_its_own_model():
assert stats is not None, "a successful Claude row must be costed, not counted as failed"
assert (stats.usage.prompt_tokens, stats.usage.completion_tokens, stats.total_tokens) == (12, 5, 17)
assert priced_models == ["claude-sonnet-4-6"]
assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((0.012, 0.010))
assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((12 * 0.001, 5 * 0.002))

View file

@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format.
import json
import urllib.parse
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from typing import Final
from types import MappingProxyType
from urllib.parse import parse_qs, urlparse
@ -16,6 +16,7 @@ from pydantic import TypeAdapter
from unittest.mock import MagicMock
import litellm
from litellm.llms.base_llm.files.transformation import BaseFileUploadStream
from litellm.llms.vertex_ai.files.transformation import (
VertexAIFilesConfig,
_get_litellm_batch_custom_id_from_labels,
@ -1995,16 +1996,18 @@ class TestVertexEmbeddingsBatchOutputTranslation:
CLAUDE_DEPLOYMENT = {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/claude-sonnet-4-6"}
def _claude_chat_entry(custom_id: str, content: str, **body: object) -> dict:
return {
"custom_id": custom_id,
def _claude_chat_entry(custom_id: str | None, content: str, **body: object) -> dict[str, object]:
entry: Final[dict[str, object]] = {
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": "claude-sonnet-4-6", "messages": [{"role": "user", "content": content}], **body},
}
return entry if custom_id is None else {"custom_id": custom_id, **entry}
def _uploaded_rows(config: VertexAIFilesConfig, litellm_params: Mapping[str, object], entries: list[dict]) -> list:
def _uploaded_rows(
config: VertexAIFilesConfig, litellm_params: Mapping[str, object], entries: Sequence[Mapping[str, object]]
) -> list:
upload = config.transform_create_file_request(
model="",
create_file_data={
@ -2014,11 +2017,14 @@ def _uploaded_rows(config: VertexAIFilesConfig, litellm_params: Mapping[str, obj
optional_params={},
litellm_params=dict(litellm_params),
)
body = b"".join(upload["streaming_media_upload"]["body_stream"].iter_bytes())
return [json.loads(line) for line in body.splitlines()]
match upload:
case {"streaming_media_upload": {"body_stream": BaseFileUploadStream() as stream}}:
return [json.loads(line) for line in b"".join(stream.iter_bytes()).splitlines()]
case _:
pytest.fail(f"expected a streamed upload, got {upload!r}")
def _claude_output_row(custom_id: str, text: str) -> dict:
def _claude_output_row(custom_id: str, text: str, content: object = None) -> dict[str, object]:
return {
"custom_id": custom_id,
"status": "",
@ -2028,7 +2034,7 @@ def _claude_output_row(custom_id: str, text: str) -> dict:
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-6",
"content": [{"type": "text", "text": text}],
"content": [{"type": "text", "text": text}] if content is None else content,
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 5},
@ -2089,8 +2095,12 @@ class TestVertexClaudeBatch:
]
def test_claude_system_message_becomes_the_anthropic_system_field(self, config):
entry = _claude_chat_entry("r1", "Hello!", max_tokens=10)
entry["body"]["messages"].insert(0, {"role": "system", "content": "be brief"})
entry = _claude_chat_entry(
"r1",
"unused",
max_tokens=10,
messages=[{"role": "system", "content": "be brief"}, {"role": "user", "content": "Hello!"}],
)
(row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry])
@ -2121,8 +2131,7 @@ class TestVertexClaudeBatch:
assert exc_info.value.status_code == 400
def test_claude_deployment_wins_over_a_gemini_body_model(self, config):
entry = _claude_chat_entry("r1", "hi", max_tokens=10)
entry["body"]["model"] = "gemini-2.5-flash"
entry = _claude_chat_entry("r1", "hi", max_tokens=10, model="gemini-2.5-flash")
(row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry])
@ -2185,6 +2194,117 @@ class TestVertexClaudeBatch:
assert rows_out["bad"]["error"]["code"] == "vertex_ai_error"
assert "too long" in rows_out["bad"]["error"]["message"]
def test_claude_row_keeps_provider_only_params_and_leaves_out_litellm_ones(self, config):
(row,) = _uploaded_rows(
config,
CLAUDE_DEPLOYMENT,
[_claude_chat_entry("r1", "hi", max_tokens=10, top_k=5, drop_params=True, metadata={"team": "a"})],
)
assert row["request"]["top_k"] == 5
assert not {"drop_params", "metadata", "extra_body"} & set(row["request"])
def test_claude_row_top_k_follows_drop_params_on_a_model_without_sampling_params(self, config, monkeypatch):
from litellm.llms.vertex_ai.common_utils import VertexAIError
monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False})
monkeypatch.setattr(litellm, "drop_params", False)
deployment = {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"}
entry = _claude_chat_entry("r1", "hi", max_tokens=10, top_k=5)
with pytest.raises(VertexAIError, match=r"line 1 .*top_k") as exc_info:
_uploaded_rows(config, deployment, [entry])
(row,) = _uploaded_rows(config, {**deployment, "drop_params": True}, [entry])
assert exc_info.value.status_code == 400
assert "top_k" not in row["request"]
@pytest.mark.parametrize("param", [{"n": 2}, {"seed": 7}, {"presence_penalty": 0.5}, {"logprobs": True}])
def test_claude_row_param_claude_does_not_support_follows_drop_params(self, config, monkeypatch, param):
from litellm.llms.vertex_ai.common_utils import VertexAIError
monkeypatch.setattr(litellm, "drop_params", False)
(name,) = param
entry = _claude_chat_entry("r1", "hi", max_tokens=10, **param)
with pytest.raises(VertexAIError, match=rf"line 1 .*{name}") as exc_info:
_uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry])
(row,) = _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "drop_params": True}, [entry])
assert exc_info.value.status_code == 400
assert name not in row["request"]
def test_claude_row_refuses_a_tool_name_claude_would_rename(self, config):
from litellm.llms.vertex_ai.common_utils import VertexAIError
def tool(name: str) -> dict[str, object]:
return {"type": "function", "function": {"name": name, "parameters": {"type": "object", "properties": {}}}}
renamed = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool("get.weather")])
kept = _claude_chat_entry("r2", "hi", max_tokens=10, tools=[tool("get_weather")])
with pytest.raises(VertexAIError, match=r"line 1 .*get\.weather") as exc_info:
_uploaded_rows(config, CLAUDE_DEPLOYMENT, [renamed])
(row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [kept])
assert exc_info.value.status_code == 400
assert [t["name"] for t in row["request"]["tools"]] == ["get_weather"]
def test_claude_row_merges_extra_body_into_the_request(self, config):
(row,) = _uploaded_rows(
config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10, extra_body={"top_k": 3})]
)
assert row["request"]["top_k"] == 3
assert "extra_body" not in row["request"]
def test_claude_response_format_row_round_trips_as_json_content(self, config):
schema = {"type": "object", "properties": {"color": {"type": "string"}}, "required": ["color"]}
(row,) = _uploaded_rows(
config,
CLAUDE_DEPLOYMENT,
[
_claude_chat_entry(
"r1",
"sky color?",
max_tokens=50,
response_format={"type": "json_schema", "json_schema": {"name": "sky", "schema": schema}},
)
],
)
assert "json_mode" not in row["request"]
(json_tool,) = row["request"]["tools"]
output_row = _claude_output_row(
"r1",
"unused",
content=[{"type": "tool_use", "id": "toolu_1", "name": json_tool["name"], "input": {"color": "blue"}}],
)
(out,) = (
json.loads(line)
for line in config._try_transform_vertex_batch_output_to_openai(json.dumps(output_row).encode()).splitlines()
)
message = out["response"]["body"]["choices"][0]["message"]
assert json.loads(message["content"]) == {"color": "blue"}
assert not message.get("tool_calls")
def test_claude_output_keeps_a_user_tool_call_as_a_tool_call(self, config):
output_row = _claude_output_row(
"r1",
"unused",
content=[{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}],
)
(out,) = (
json.loads(line)
for line in config._try_transform_vertex_batch_output_to_openai(json.dumps(output_row).encode()).splitlines()
)
(tool_call,) = out["response"]["body"]["choices"][0]["message"]["tool_calls"]
assert tool_call["function"]["name"] == "get_weather"
assert json.loads(tool_call["function"]["arguments"]) == {"city": "Paris"}
def test_claude_input_file_read_back_is_returned_unchanged(self, config):
(row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10)])
content = json.dumps(row).encode()
@ -2192,8 +2312,7 @@ class TestVertexClaudeBatch:
assert config._try_transform_vertex_batch_output_to_openai(content) == content
def test_malformed_claude_message_fails_only_its_own_row(self, config):
malformed = _claude_output_row("bad", "unused")
malformed["response"]["content"] = 5
malformed = _claude_output_row("bad", "unused", content=5)
content = json.dumps(_claude_output_row("ok", "fine")).encode() + b"\n" + json.dumps(malformed).encode()
ok_row, bad_row = (
@ -2206,8 +2325,7 @@ class TestVertexClaudeBatch:
assert bad_row["error"]["code"] == "transformation_error"
def test_claude_row_without_custom_id_has_no_custom_id_field(self, config):
entry = _claude_chat_entry("unused", "hi", max_tokens=10)
del entry["custom_id"]
entry = _claude_chat_entry(None, "hi", max_tokens=10)
(row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry])