scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 98

fix(vertex_ai): translate /v1/responses batch rows through the Responses-to-Chat bridge (#43042)

Browse source 
* fix(vertex_ai): translate /v1/responses batch rows through the Responses-to-Chat bridge

Vertex batch uploads treated every non-embeddings JSONL row as a chat
completions body, so a /v1/responses row lost its input and reached GCS
as a blank text part. Route detection now recognizes /v1/responses rows
and bridges them to chat through the same Responses-to-Chat bridge the
real-time path uses. That bridge call moves out of the Bedrock files
transformation into a shared helper both providers call, forwarding the
record's fields as sent, like real time, instead of validating them
against the SDK TypedDicts whose required keys clients omit.

* chore(batches): type the Vertex responses test helper and drop the quoted input cast

* fix(batches): translate developer messages to system on Vertex and Bedrock batch rows like real time

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-24 16:15:58 -07:00 • committed by GitHub
parent ba77646991
commit 3fa688223d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 280 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
litellm/llms/base_llm/files/batch_records.py Normal file
View file

@ -0,0 +1,54 @@
from collections.abc import Iterable, Mapping
from functools import cache
from types import MappingProxyType
from typing import Final, cast, get_type_hints
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams
def _frozen_mapping(items: Iterable[tuple[str, object]]) -> Mapping[str, object]:
return MappingProxyType(dict(items))
@cache
def _responses_request_keys() -> frozenset[str]:
return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams))
def responses_batch_body_to_chat_body(
openai_request_body: Mapping[str, object],
custom_llm_provider: str | None = None,
) -> dict[str, object]: # mutable-ok: provider transforms take the bridged chat body as a plain dict
"""
Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body.
Batch providers translate chat bodies into their own request shape, so a Responses
record goes through the same Responses-to-Chat bridge the real-time path uses for
providers without a native Responses API: `input`, `instructions`, `max_output_tokens`
and the tool params translate identically in batch and real time. Like real time, the
record's fields are forwarded as sent instead of validated against the SDK TypedDicts,
whose required keys (a function tool's `strict`, an image part's `detail`) clients omit.
"""
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
responses_input: Final = openai_request_body.get("input")
if responses_input is None:
raise ValueError(
"Batch record for /v1/responses is missing required `input` field: "
f"model={openai_request_body.get('model', '')}"
)
model: Final = openai_request_body.get("model")
chat_input: Final = cast(str | ResponseInputParam, responses_input) # cast-ok: forwarded as sent
responses_request: Final = cast( # cast-ok: client-supplied fields forwarded verbatim, as real time does
ResponsesAPIOptionalRequestParams,
_frozen_mapping((key, value) for key, value in openai_request_body.items() if key in _responses_request_keys()),
)
return LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # transformer declares a bare dict return
model=model if isinstance(model, str) else "",
input=chat_input,
responses_api_request=responses_request,
custom_llm_provider=custom_llm_provider,
metadata=openai_request_body.get("metadata"),
)

53
litellm/llms/bedrock/files/transformation.py
View file

@ -8,7 +8,6 @@ from collections.abc import Iterable, Mapping, MutableMapping, Sequence
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime
from functools import cache
from itertools import chain
from types import MappingProxyType
from typing import Any, Final, Literal, TypeAlias, TypedDict
@ -17,7 +16,7 @@ from urllib.parse import quote, unquote, urlencode
import httpx
from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
from pydantic import BaseModel, ConfigDict, Field
from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
@ -41,7 +40,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_data,
text_completion_prompt_to_messages,
)
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
LiteLLMLoggingObj,
@ -56,8 +57,6 @@ from litellm.types.llms.openai import (
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
PathLike,
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
)
from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums
from litellm.utils import get_llm_provider
@ -130,22 +129,6 @@ class _S3UploadResponse(TypedDict, total=False):
ContentLength: ReadOnly[int]
# JSONL batch records are untyped json, so the `/v1/responses` fields are
# validated into their concrete Responses API types before being handed to the
# Responses-to-Chat bridge. Both adapters drop keys the Responses API doesn't
# define, which is what the bridge would ignore anyway. Built on first use
# rather than at import: `ResponseInputParam` is a deep union and only batch
# files carrying `/v1/responses` records need it.
@cache
def _responses_input_adapter() -> TypeAdapter[str | ResponseInputParam]:
return TypeAdapter(str | ResponseInputParam)
@cache
def _responses_request_adapter() -> TypeAdapter[ResponsesAPIOptionalRequestParams]:
return TypeAdapter(ResponsesAPIOptionalRequestParams)
class _BedrockS3RequestParams(AwsAuthParams):
"""Typed view of the credential/region params the S3 GetObject path reads."""
@ -859,33 +842,9 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
Delegates to the same Responses-to-Chat bridge the real-time path uses
for providers without a native Responses API (which is every Bedrock
model), so `input`, `instructions`, `max_output_tokens` and the tool
params translate identically in batch and real time. The bridge always
emits a `tools` key; an empty one is dropped rather than shipped as an
empty array inside `modelInput`.
params translate identically in batch and real time.
"""
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
responses_input: Final = openai_request_body.get("input")
if responses_input is None:
raise ValueError(
"Batch record for /v1/responses is missing required `input` field: "
f"model={openai_request_body.get('model', '')}"
)
chat_body: Final[Mapping[str, object]] = (
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=openai_request_body.get("model", ""),
input=_responses_input_adapter().validate_python(responses_input),
responses_api_request=_responses_request_adapter().validate_python(
_frozen_mapping(
(key, value) for key, value in openai_request_body.items() if key not in ("model", "input")
)
),
metadata=openai_request_body.get("metadata"),
)
)
return _frozen_mapping((key, value) for key, value in chat_body.items() if key != "tools" or value)
return responses_batch_body_to_chat_body(openai_request_body)
@staticmethod
def _transform_batch_body_to_chat_body(
@ -922,7 +881,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
"""
from litellm.types.utils import LlmProviders
messages: Final = openai_request_body.get("messages", [])
messages: Final = map_developer_role_to_system_role(openai_request_body.get("messages", []))
optional_params: Final = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
# --- Anthropic: use existing AmazonAnthropicClaudeConfig ---

32
litellm/llms/vertex_ai/files/transformation.py
View file

@ -35,7 +35,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_data,
extract_file_metadata,
)
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
BaseFileUploadStream,
@ -529,21 +531,30 @@ def is_passthrough_batch_upload(create_file_data: Mapping[str, object], litellm_
return create_file_data.get("purpose") == "batch" and litellm_params.get("passthrough") is True
def _is_embeddings_batch_entry(openai_entry: Mapping[str, object]) -> bool:
def _batch_entry_route_path(openai_entry: Mapping[str, object]) -> str:
"""
Whether an OpenAI batch JSONL line targets the embeddings endpoint.
The route an OpenAI batch JSONL line targets, without query string or trailing slash.
OpenAI puts the target route on each line's `url` (e.g. `/v1/embeddings`); Vertex
has no equivalent per-line field, so the route decides which Vertex request shape
the line has to be translated into.
"""
url = openai_entry.get("url")
url: Final = openai_entry.get("url")
if not isinstance(url, str):
return False
path = url.split("?")[0].rstrip("/")
return ""
return url.split("?")[0].rstrip("/")
def _is_embeddings_batch_entry(openai_entry: Mapping[str, object]) -> bool:
path: Final = _batch_entry_route_path(openai_entry)
return path == "embeddings" or path.endswith("/embeddings")
def _is_responses_batch_entry(openai_entry: Mapping[str, object]) -> bool:
path: Final = _batch_entry_route_path(openai_entry)
return path == "responses" or path.endswith("/responses")
def _openai_embedding_input_elements(
embedding_input: GeminiEmbeddingInput,
) -> tuple[str | list[str], ...]:
@ -665,10 +676,15 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry)
openai_request_body: Final = openai_entry.get("body") or {}
chat_request_body: Final = (
responses_batch_body_to_chat_body(openai_request_body, custom_llm_provider="vertex_ai")
if _is_responses_batch_entry(openai_entry)
else openai_request_body
)
vertex_request_body: Final = _transform_request_body(
messages=openai_request_body.get("messages", []),
model=openai_request_body.get("model", ""),
optional_params=map_openai_to_vertex_params(openai_request_body),
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
model=chat_request_body.get("model", ""),
optional_params=map_openai_to_vertex_params(chat_request_body),
custom_llm_provider="vertex_ai",
litellm_params={},
cached_content=None,

43
tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py
View file

@ -1672,6 +1672,49 @@ class TestBedrockBatchNonChatEndpointRecords:
assert "input" not in model_input
assert "max_output_tokens" not in model_input
def test_anthropic_responses_record_accepts_a_function_tool_without_strict(self):
"""Clients omit the SDK's required `strict`; the record is forwarded like real time, not validated."""
parameters = {"type": "object", "properties": {"city": {"type": "string"}}}
model_input = self._transform(
{
"custom_id": "4a",
"method": "POST",
"url": "/v1/responses",
"body": {
"model": self.ANTHROPIC_MODEL,
"input": "Weather in Paris?",
"tools": [{"type": "function", "name": "get_weather", "parameters": parameters}],
},
}
)
assert model_input["messages"][0]["content"] == [{"type": "text", "text": "Weather in Paris?"}]
tool = model_input["tools"][0]
function = tool.get("function", tool)
assert (function["name"], function.get("parameters", function.get("input_schema"))) == ("get_weather", parameters)
@pytest.mark.parametrize(
("url", "body"),
[
(
"/v1/responses",
{"input": [{"role": "developer", "content": "be terse"}, {"role": "user", "content": "ping"}]},
),
(
"/v1/chat/completions",
{"messages": [{"role": "developer", "content": "be terse"}, {"role": "user", "content": "ping"}]},
),
],
ids=["responses", "chat"],
)
def test_anthropic_developer_role_becomes_the_system_prompt_like_real_time(self, url, body):
model_input = self._transform(
{"custom_id": "4c", "method": "POST", "url": url, "body": {"model": self.ANTHROPIC_MODEL, **body}}
)
assert model_input["system"] == [{"type": "text", "text": "be terse"}]
assert [message["role"] for message in model_input["messages"]] == ["user"]
def test_responses_record_keeps_metadata(self):
"""`metadata` reaches the bridge, which reads it as its own kwarg."""
model_input = self._transform(

153
tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py
View file

@ -5,6 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format.
import json
import urllib.parse
from collections.abc import Mapping
from types import MappingProxyType
from urllib.parse import parse_qs, urlparse
@ -1447,6 +1448,158 @@ class TestVertexEmbeddingsBatchInputTranslation:
assert "content" in embeddings_row["request"]
def _responses_entry(
body: Mapping[str, object] | None = None,
custom_id: str = "resp-1",
url: str = "/v1/responses",
) -> dict[str, object]:
return {
"custom_id": custom_id,
"method": "POST",
"url": url,
"body": body
if body is not None
else {"model": "gemini-2.5-flash", "input": "What was the top headline in world news yesterday?"},
}
class TestVertexResponsesBatchInputTranslation:
"""
/v1/responses batch lines carry `input`, not `messages`, so they go through the
Responses-to-Chat bridge before the Gemini translation instead of uploading as an
empty text part.
"""
def test_string_input_becomes_the_user_prompt(self):
(row,) = _wrap_entries([_responses_entry()])
assert row["request"]["contents"] == [
{"role": "user", "parts": [{"text": "What was the top headline in world news yesterday?"}]}
]
assert row["request"]["labels"]["litellm_custom_id"] == "resp-1"
def test_instructions_and_input_items_map_like_real_time(self):
(row,) = _wrap_entries(
[
_responses_entry(
body={
"model": "gemini-2.5-flash",
"instructions": "be terse",
"input": [
{"role": "user", "content": "what is 2+2?"},
{"role": "assistant", "content": "4"},
{"role": "user", "content": "and 3+3?"},
],
"max_output_tokens": 32,
"temperature": 0.2,
}
)
]
)
request = row["request"]
assert request["system_instruction"] == {"parts": [{"text": "be terse"}]}
assert [content["role"] for content in request["contents"]] == ["user", "model", "user"]
assert request["contents"][-1]["parts"] == [{"text": "and 3+3?"}]
assert request["generationConfig"]["max_output_tokens"] == 32
assert request["generationConfig"]["temperature"] == 0.2
def test_web_search_tool_keeps_the_prompt(self):
(row,) = _wrap_entries(
[
_responses_entry(
body={
"model": "gemini-2.5-flash",
"input": "What was the top headline in world news yesterday?",
"tools": [{"type": "web_search"}],
}
)
]
)
assert row["request"]["contents"] == [
{"role": "user", "parts": [{"text": "What was the top headline in world news yesterday?"}]}
]
assert row["request"]["tools"]
def test_sdk_optional_keys_are_not_required_like_real_time(self):
(row,) = _wrap_entries(
[
_responses_entry(
body={
"model": "gemini-2.5-flash",
"input": [
{
"role": "user",
"content": [
{"type": "input_text", "text": "Weather in the pictured city?"},
{"type": "input_image", "image_url": "https://example.com/paris.png"},
],
}
],
"tools": [
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
}
],
}
)
]
)
request = row["request"]
assert request["contents"][0]["parts"] == [
{"text": "Weather in the pictured city?"},
{"file_data": {"mime_type": "image/png", "file_uri": "https://example.com/paris.png"}},
]
assert request["tools"][0]["function_declarations"][0]["name"] == "get_weather"
@pytest.mark.parametrize(
"url",
["/v1/responses", "/v1/responses/", "/v1/responses?beta=1", "responses", "https://api.openai.com/v1/responses"],
)
def test_route_spellings_are_all_responses(self, url):
(row,) = _wrap_entries([_responses_entry(url=url)])
assert row["request"]["contents"][0]["parts"] == [
{"text": "What was the top headline in world news yesterday?"}
]
def test_missing_input_fails_the_upload(self):
with pytest.raises(ValueError, match="missing required `input` field"):
_wrap_entries([_responses_entry(body={"model": "gemini-2.5-flash"})])
@pytest.mark.parametrize(
"entry",
[
_responses_entry(
body={
"model": "gemini-2.5-flash",
"input": [{"role": "developer", "content": "be terse"}, {"role": "user", "content": "ping"}],
}
),
{
"custom_id": "chat-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {
"model": "gemini-2.5-flash",
"messages": [{"role": "developer", "content": "be terse"}, {"role": "user", "content": "ping"}],
},
},
],
ids=["responses", "chat"],
)
def test_developer_role_becomes_the_system_instruction_like_real_time(self, entry):
(row,) = _wrap_entries([entry])
request = row["request"]
assert request["system_instruction"] == {"parts": [{"text": "be terse"}]}
assert request["contents"] == [{"role": "user", "parts": [{"text": "ping"}]}]
class TestVertexEmbeddingsBatchOutputTranslation:
"""Vertex Gemini Embedding batch output rows must come back as OpenAI batch rows."""