mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(responses)!: answer 400 for a bridged input item without a role
An input item with no role reached a bridged model as a user turn, and OpenAI answers 400 for the same request. The bridge now rejects it before any provider call, naming input[N].role, on inference, batch uploads and input token counts. Role-less shapes OpenAI accepts, like a bare item reference, still go through
This commit is contained in:
parent
4c8b90945f
commit
a48cc02eae
11 changed files with 228 additions and 2 deletions
|
|
@ -32,6 +32,7 @@ def responses_batch_body_to_chat_body(
|
|||
"""
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
reject_input_item_without_role,
|
||||
)
|
||||
|
||||
responses_input: Final = openai_request_body.get("input")
|
||||
|
|
@ -41,6 +42,7 @@ def responses_batch_body_to_chat_body(
|
|||
f"model={openai_request_body.get('model', '')}"
|
||||
)
|
||||
model: Final = openai_request_body.get("model")
|
||||
reject_input_item_without_role(responses_input, model if isinstance(model, str) else "", custom_llm_provider)
|
||||
chat_input: Final = cast(str | ResponseInputParam, responses_input) # cast-ok: forwarded as sent
|
||||
responses_request: Final = cast( # cast-ok: client-supplied fields forwarded verbatim, as real time does
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
|
|
|
|||
|
|
@ -15503,7 +15503,7 @@ async def _count_request_tokens(
|
|||
)
|
||||
)
|
||||
if isinstance(chat_count, ChatCountBridgeError):
|
||||
raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400)
|
||||
raise ProxyException(message=chat_count.message, type="invalid_request_error", param=chat_count.param, code=400)
|
||||
ladder: Final = (
|
||||
None
|
||||
if provider_counter is None
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
|
|||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
ChatCompletionSession,
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
reject_input_item_without_role,
|
||||
)
|
||||
from litellm.responses.main import previous_response_client_ids
|
||||
from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam
|
||||
|
|
@ -51,6 +52,7 @@ class ChatCountRequest:
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class ChatCountBridgeError:
|
||||
message: str
|
||||
param: str | None = None
|
||||
|
||||
|
||||
def _plain(item: object) -> object:
|
||||
|
|
@ -193,6 +195,7 @@ async def chat_count_request(
|
|||
):
|
||||
return None
|
||||
try:
|
||||
reject_input_item_without_role(responses_request.get("input"), model, custom_llm_provider)
|
||||
chat_body: Final = await _with_previous_turns(
|
||||
responses_batch_body_to_chat_body(
|
||||
_with_compaction_as_summary_text(
|
||||
|
|
@ -216,6 +219,6 @@ async def chat_count_request(
|
|||
params=_counted_params(chat_body, tools, policy),
|
||||
)
|
||||
except BadRequestError as e:
|
||||
return ChatCountBridgeError(message=str(e.message))
|
||||
return ChatCountBridgeError(message=str(e.message), param=e.param)
|
||||
except (ValidationError, ValueError, KeyError) as e:
|
||||
return ChatCountBridgeError(message=f"Invalid request for token counting: {e}")
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo
|
|||
from litellm.responses.litellm_completion_transformation.tool_search import has_client_tool_search
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
LiteLLMCompletionResponsesConfig,
|
||||
reject_input_item_without_role,
|
||||
with_compaction_items_as_params,
|
||||
)
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
|
@ -74,6 +75,7 @@ class LiteLLMCompletionTransformationHandler:
|
|||
| BaseResponsesAPIStreamingIterator
|
||||
| Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
|
||||
):
|
||||
reject_input_item_without_role(input, model, custom_llm_provider)
|
||||
hoisted: Final = hoist_additional_tools(
|
||||
input,
|
||||
responses_api_request.get("tools"),
|
||||
|
|
|
|||
|
|
@ -436,6 +436,39 @@ def _input_item_role(input_item: Mapping[str, object]) -> str:
|
|||
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
|
||||
|
||||
|
||||
def _lacks_required_role(item: object) -> bool:
|
||||
if not isinstance(item, Mapping):
|
||||
return False
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(item)
|
||||
if fields.get("role"):
|
||||
return False
|
||||
item_type: Final = fields.get("type")
|
||||
if item_type is None:
|
||||
is_item_reference: Final = bool(fields.get("id")) and fields.get("content") is None
|
||||
return not is_item_reference
|
||||
return item_type == "message"
|
||||
|
||||
|
||||
def _first_item_without_role(input: object) -> int | None:
|
||||
if not isinstance(input, list):
|
||||
return None
|
||||
items: Final = _OBJECT_LIST_ADAPTER.validate_python(input)
|
||||
return next((index for index, item in enumerate(items) if _lacks_required_role(item)), None)
|
||||
|
||||
|
||||
def reject_input_item_without_role(input: object, model: str, custom_llm_provider: str | None) -> None:
|
||||
index: Final = _first_item_without_role(input)
|
||||
if index is None:
|
||||
return
|
||||
param: Final = f"input[{index}].role"
|
||||
raise litellm.BadRequestError(
|
||||
message=f"Missing required parameter: '{param}'.",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider or "",
|
||||
body={"type": "invalid_request_error", "param": param, "code": "missing_required_parameter"},
|
||||
)
|
||||
|
||||
|
||||
SessionMessage: TypeAlias = (
|
||||
AllMessageValues
|
||||
| GenericChatCompletionMessage
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_BRIDGED_MODEL: Final = "deepseek/gpt-4o-mini"
|
||||
_USER_TURN: Final[dict[str, JsonValue]] = {"role": "user", "content": "hello"}
|
||||
|
||||
|
||||
def _completion(_: Request) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-input-without-role",
|
||||
"object": "chat.completion",
|
||||
"created": 1789788254,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}},
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
[
|
||||
pytest.param({"content": "no role"}, id="content-only"),
|
||||
pytest.param({"type": "message", "content": "no role"}, id="typed-message"),
|
||||
pytest.param({"role": None, "content": "no role"}, id="null-role"),
|
||||
pytest.param({"role": "", "content": "no role"}, id="empty-role"),
|
||||
],
|
||||
)
|
||||
def test_bridged_input_item_without_a_role_answers_400_and_reaches_no_provider(
|
||||
gateway: Gateway, item: dict[str, JsonValue]
|
||||
) -> None:
|
||||
with wire_server(_completion) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=wire.url + "/v1")
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": [_USER_TURN, item], "store": False}
|
||||
)
|
||||
|
||||
assert response.status_code == 400, response.text
|
||||
error: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(response.content)["error"])
|
||||
assert error["type"] == "invalid_request_error", response.text
|
||||
assert error["param"] == "input[1].role", response.text
|
||||
assert "input[1].role" in str(error["message"]), response.text
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def test_bridged_role_less_item_reference_still_reaches_the_provider(gateway: Gateway) -> None:
|
||||
with wire_server(_completion) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=wire.url + "/v1")
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": [{"id": "msg_1"}, _USER_TURN], "store": False}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
@ -4516,6 +4516,19 @@ def _batch_line_with_body(field: str, value: object) -> str:
|
|||
"Batch input file line 2 could not be converted to a Bedrock batch record",
|
||||
id="string-messages",
|
||||
),
|
||||
pytest.param(
|
||||
json.dumps(
|
||||
{
|
||||
**_VALID_BATCH_RECORD,
|
||||
"url": "/v1/responses",
|
||||
"body": {"model": _VALID_BATCH_RECORD["body"]["model"], "input": [{"content": "no role"}]},
|
||||
}
|
||||
).encode()
|
||||
+ b"\n",
|
||||
"Batch input file line 1 could not be converted to a Bedrock batch record: "
|
||||
"litellm.BadRequestError: Missing required parameter: 'input[0].role'.",
|
||||
id="responses-item-without-role",
|
||||
),
|
||||
pytest.param(
|
||||
f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(),
|
||||
"Batch input file line 2 could not be parsed",
|
||||
|
|
|
|||
|
|
@ -1624,6 +1624,19 @@ class TestVertexResponsesBatchInputTranslation:
|
|||
with pytest.raises(ValueError, match="missing required `input` field"):
|
||||
_wrap_entries([_responses_entry(body={"model": "gemini-2.5-flash"})])
|
||||
|
||||
def test_item_without_a_role_fails_the_upload_naming_it(self):
|
||||
with pytest.raises(litellm.BadRequestError, match=r"input\[1\]\.role"):
|
||||
_wrap_entries(
|
||||
[
|
||||
_responses_entry(
|
||||
body={
|
||||
"model": "gemini-2.5-flash",
|
||||
"input": [{"role": "user", "content": "ping"}, {"content": "no role"}],
|
||||
}
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"entry",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -3984,6 +3984,14 @@ async def test_responses_count_on_claude_rejects_forced_tool_choice_without_drop
|
|||
assert (rejected.value.code, "forced tool use" in rejected.value.message) == ("400", True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_count_on_claude_names_an_input_item_without_a_role(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
with pytest.raises(ProxyException) as rejected:
|
||||
await _claude_count(monkeypatch, {}, {"input": [{"content": "Weather?"}]})
|
||||
|
||||
assert (rejected.value.code, rejected.value.param) == ("400", "input[0].role")
|
||||
|
||||
|
||||
def _texts(message: JsonValue) -> list[str]:
|
||||
assert isinstance(message, dict)
|
||||
content: Final = message["content"]
|
||||
|
|
|
|||
|
|
@ -129,6 +129,18 @@ async def test_claude_count_request_rejects_a_malformed_tool_flag_without_drop_p
|
|||
assert "defer_loading" in flagged.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_count_request_rejects_an_item_without_a_role_at_its_index_in_the_callers_input() -> None:
|
||||
rejected: Final = await _claude_count(
|
||||
{"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, {"content": "Weather in SF?"}]},
|
||||
DropPolicy(drop_params=False, additional_drop_params=frozenset()),
|
||||
)
|
||||
|
||||
assert isinstance(rejected, ChatCountBridgeError)
|
||||
assert rejected.param == "input[1].role"
|
||||
assert "input[1].role" in rejected.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> None:
|
||||
policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset({"tools"}))
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.responses.litellm_completion_transformation.handler import (
|
|||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD
|
||||
from openai.types.responses import ResponseCompactionItem
|
||||
|
||||
|
||||
class _StopForwarding(Exception):
|
||||
|
|
@ -310,3 +311,77 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
|
|||
("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
|
||||
("user", json.dumps([{"type": "text", "text": "Double it"}])),
|
||||
]
|
||||
|
||||
|
||||
_USER_TURN: Final = {"role": "user", "content": "hello"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_items", "param"),
|
||||
[
|
||||
([{"content": "hi"}], "input[0].role"),
|
||||
([{"role": None, "content": "hi"}], "input[0].role"),
|
||||
([{"role": "", "content": "hi"}], "input[0].role"),
|
||||
([{"type": "message", "content": "hi"}], "input[0].role"),
|
||||
([{"type": "message", "id": "msg_1"}], "input[0].role"),
|
||||
([{"content": [{"type": "input_text", "text": "hi"}]}], "input[0].role"),
|
||||
([{"content": None}], "input[0].role"),
|
||||
([{}], "input[0].role"),
|
||||
([{"id": "msg_1", "content": "hi"}], "input[0].role"),
|
||||
([_USER_TURN, {"content": "bye"}], "input[1].role"),
|
||||
([{"type": "additional_tools", "tools": []}, {"content": "hi"}], "input[1].role"),
|
||||
],
|
||||
)
|
||||
def test_input_item_without_role_answers_400_naming_it_and_sends_nothing(input_items, param):
|
||||
forwarded: Final[list[object]] = []
|
||||
|
||||
with patch("litellm.completion", lambda **kwargs: forwarded.append(kwargs)):
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
LiteLLMCompletionTransformationHandler().response_api_handler(
|
||||
model="claude-sonnet-5-5",
|
||||
input=input_items,
|
||||
responses_api_request={},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert forwarded == []
|
||||
assert raised.value.status_code == 400
|
||||
assert raised.value.param == param
|
||||
assert raised.value.type == "invalid_request_error"
|
||||
assert raised.value.code == "missing_required_parameter"
|
||||
assert param in str(raised.value)
|
||||
assert raised.value.llm_provider == "anthropic"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_items",
|
||||
[
|
||||
"hello",
|
||||
[_USER_TURN],
|
||||
[_USER_TURN, {"id": "msg_1"}],
|
||||
[_USER_TURN, {"type": "item_reference", "id": "msg_1"}],
|
||||
[_USER_TURN, {"type": "reasoning", "id": "rs_1", "summary": []}],
|
||||
[
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "done"},
|
||||
],
|
||||
[ResponseCompactionItem(id="cmp_1", encrypted_content="opaque", type="compaction"), _USER_TURN],
|
||||
],
|
||||
)
|
||||
def test_role_less_shapes_openai_accepts_still_reach_the_provider(input_items):
|
||||
captured: Final[dict[str, object]] = {}
|
||||
|
||||
def fake_completion(**kwargs):
|
||||
captured.update(kwargs)
|
||||
raise _StopForwarding()
|
||||
|
||||
with patch("litellm.completion", fake_completion):
|
||||
with pytest.raises(_StopForwarding):
|
||||
LiteLLMCompletionTransformationHandler().response_api_handler(
|
||||
model="claude-sonnet-5-5",
|
||||
input=input_items,
|
||||
responses_api_request={},
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
assert captured["messages"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue