diff --git a/litellm/llms/base_llm/files/batch_records.py b/litellm/llms/base_llm/files/batch_records.py index 070cf73cd57..ab1a5fe296f 100644 --- a/litellm/llms/base_llm/files/batch_records.py +++ b/litellm/llms/base_llm/files/batch_records.py @@ -32,6 +32,7 @@ def responses_batch_body_to_chat_body( """ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, + reject_input_item_without_role, ) responses_input: Final = openai_request_body.get("input") @@ -41,6 +42,7 @@ def responses_batch_body_to_chat_body( f"model={openai_request_body.get('model', '')}" ) model: Final = openai_request_body.get("model") + reject_input_item_without_role(responses_input, model if isinstance(model, str) else "", custom_llm_provider) chat_input: Final = cast(str | ResponseInputParam, responses_input) # cast-ok: forwarded as sent responses_request: Final = cast( # cast-ok: client-supplied fields forwarded verbatim, as real time does ResponsesAPIOptionalRequestParams, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b0ccf74c63b..728da8a2472 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15503,7 +15503,7 @@ async def _count_request_tokens( ) ) if isinstance(chat_count, ChatCountBridgeError): - raise ProxyException(message=chat_count.message, type="invalid_request_error", param=None, code=400) + raise ProxyException(message=chat_count.message, type="invalid_request_error", param=chat_count.param, code=400) ladder: Final = ( None if provider_counter is None diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index a24a26aac42..d8d5a5ba6a4 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -18,6 +18,7 @@ from litellm.responses.litellm_completion_transformation.compaction_items import from litellm.responses.litellm_completion_transformation.transformation import ( ChatCompletionSession, LiteLLMCompletionResponsesConfig, + reject_input_item_without_role, ) from litellm.responses.main import previous_response_client_ids from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam @@ -51,6 +52,7 @@ class ChatCountRequest: @dataclass(frozen=True, slots=True) class ChatCountBridgeError: message: str + param: str | None = None def _plain(item: object) -> object: @@ -193,6 +195,7 @@ async def chat_count_request( ): return None try: + reject_input_item_without_role(responses_request.get("input"), model, custom_llm_provider) chat_body: Final = await _with_previous_turns( responses_batch_body_to_chat_body( _with_compaction_as_summary_text( @@ -216,6 +219,6 @@ async def chat_count_request( params=_counted_params(chat_body, tools, policy), ) except BadRequestError as e: - return ChatCountBridgeError(message=str(e.message)) + return ChatCountBridgeError(message=str(e.message), param=e.param) except (ValidationError, ValueError, KeyError) as e: return ChatCountBridgeError(message=f"Invalid request for token counting: {e}") diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 1f56ddf863e..df0c2cf1150 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -20,6 +20,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo from litellm.responses.litellm_completion_transformation.tool_search import has_client_tool_search from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, + reject_input_item_without_role, with_compaction_items_as_params, ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -74,6 +75,7 @@ class LiteLLMCompletionTransformationHandler: | BaseResponsesAPIStreamingIterator | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator] ): + reject_input_item_without_role(input, model, custom_llm_provider) hoisted: Final = hoist_additional_tools( input, responses_api_request.get("tools"), diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index ce8841c9190..ea3a73b1866 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -436,6 +436,39 @@ def _input_item_role(input_item: Mapping[str, object]) -> str: return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated +def _lacks_required_role(item: object) -> bool: + if not isinstance(item, Mapping): + return False + fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(item) + if fields.get("role"): + return False + item_type: Final = fields.get("type") + if item_type is None: + is_item_reference: Final = bool(fields.get("id")) and fields.get("content") is None + return not is_item_reference + return item_type == "message" + + +def _first_item_without_role(input: object) -> int | None: + if not isinstance(input, list): + return None + items: Final = _OBJECT_LIST_ADAPTER.validate_python(input) + return next((index for index, item in enumerate(items) if _lacks_required_role(item)), None) + + +def reject_input_item_without_role(input: object, model: str, custom_llm_provider: str | None) -> None: + index: Final = _first_item_without_role(input) + if index is None: + return + param: Final = f"input[{index}].role" + raise litellm.BadRequestError( + message=f"Missing required parameter: '{param}'.", + model=model, + llm_provider=custom_llm_provider or "", + body={"type": "invalid_request_error", "param": param, "code": "missing_required_parameter"}, + ) + + SessionMessage: TypeAlias = ( AllMessageValues | GenericChatCompletionMessage diff --git a/tests/integration/providers/test_responses_bridge_input_without_role_wire.py b/tests/integration/providers/test_responses_bridge_input_without_role_wire.py new file mode 100644 index 00000000000..93cf9ad0ea1 --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_input_without_role_wire.py @@ -0,0 +1,65 @@ +import json +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BRIDGED_MODEL: Final = "deepseek/gpt-4o-mini" +_USER_TURN: Final[dict[str, JsonValue]] = {"role": "user", "content": "hello"} + + +def _completion(_: Request) -> Reply: + return Reply( + body=json.dumps( + { + "id": "chatcmpl-input-without-role", + "object": "chat.completion", + "created": 1789788254, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}, + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5}, + } + ).encode() + ) + + +@pytest.mark.parametrize( + "item", + [ + pytest.param({"content": "no role"}, id="content-only"), + pytest.param({"type": "message", "content": "no role"}, id="typed-message"), + pytest.param({"role": None, "content": "no role"}, id="null-role"), + pytest.param({"role": "", "content": "no role"}, id="empty-role"), + ], +) +def test_bridged_input_item_without_a_role_answers_400_and_reaches_no_provider( + gateway: Gateway, item: dict[str, JsonValue] +) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=wire.url + "/v1") + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": [_USER_TURN, item], "store": False} + ) + + assert response.status_code == 400, response.text + error: Final = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_json(response.content)["error"]) + assert error["type"] == "invalid_request_error", response.text + assert error["param"] == "input[1].role", response.text + assert "input[1].role" in str(error["message"]), response.text + assert wire.drain() == () + + +def test_bridged_role_less_item_reference_still_reaches_the_provider(gateway: Gateway) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=wire.url + "/v1") + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": [{"id": "msg_1"}, _USER_TURN], "store": False} + ) + + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 9d9e39df944..6fcbf5fdf2a 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -4516,6 +4516,19 @@ def _batch_line_with_body(field: str, value: object) -> str: "Batch input file line 2 could not be converted to a Bedrock batch record", id="string-messages", ), + pytest.param( + json.dumps( + { + **_VALID_BATCH_RECORD, + "url": "/v1/responses", + "body": {"model": _VALID_BATCH_RECORD["body"]["model"], "input": [{"content": "no role"}]}, + } + ).encode() + + b"\n", + "Batch input file line 1 could not be converted to a Bedrock batch record: " + "litellm.BadRequestError: Missing required parameter: 'input[0].role'.", + id="responses-item-without-role", + ), pytest.param( f"{_VALID_BATCH_LINE}\n{'[' * 1_000_000}{']' * 1_000_000}\n".encode(), "Batch input file line 2 could not be parsed", diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 98e204dd44e..4246cdd5249 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -1624,6 +1624,19 @@ class TestVertexResponsesBatchInputTranslation: with pytest.raises(ValueError, match="missing required `input` field"): _wrap_entries([_responses_entry(body={"model": "gemini-2.5-flash"})]) + def test_item_without_a_role_fails_the_upload_naming_it(self): + with pytest.raises(litellm.BadRequestError, match=r"input\[1\]\.role"): + _wrap_entries( + [ + _responses_entry( + body={ + "model": "gemini-2.5-flash", + "input": [{"role": "user", "content": "ping"}, {"content": "no role"}], + } + ) + ] + ) + @pytest.mark.parametrize( "entry", [ diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index c182cd1ec75..c731bee98ec 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3984,6 +3984,14 @@ async def test_responses_count_on_claude_rejects_forced_tool_choice_without_drop assert (rejected.value.code, "forced tool use" in rejected.value.message) == ("400", True) +@pytest.mark.asyncio +async def test_responses_count_on_claude_names_an_input_item_without_a_role(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(ProxyException) as rejected: + await _claude_count(monkeypatch, {}, {"input": [{"content": "Weather?"}]}) + + assert (rejected.value.code, rejected.value.param) == ("400", "input[0].role") + + def _texts(message: JsonValue) -> list[str]: assert isinstance(message, dict) content: Final = message["content"] diff --git a/tests/unit/responses/litellm_completion_transformation/test_count_request.py b/tests/unit/responses/litellm_completion_transformation/test_count_request.py index 2080a091160..bb212bffc3d 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_count_request.py +++ b/tests/unit/responses/litellm_completion_transformation/test_count_request.py @@ -129,6 +129,18 @@ async def test_claude_count_request_rejects_a_malformed_tool_flag_without_drop_p assert "defer_loading" in flagged.message +@pytest.mark.asyncio +async def test_claude_count_request_rejects_an_item_without_a_role_at_its_index_in_the_callers_input() -> None: + rejected: Final = await _claude_count( + {"input": [{"type": "additional_tools", "tools": [_WEATHER_TOOL]}, {"content": "Weather in SF?"}]}, + DropPolicy(drop_params=False, additional_drop_params=frozenset()), + ) + + assert isinstance(rejected, ChatCountBridgeError) + assert rejected.param == "input[1].role" + assert "input[1].role" in rejected.message + + @pytest.mark.asyncio async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> None: policy: Final = DropPolicy(drop_params=False, additional_drop_params=frozenset({"tools"})) diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..cb786b23b7e 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -29,6 +29,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD +from openai.types.responses import ResponseCompactionItem class _StopForwarding(Exception): @@ -310,3 +311,77 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), ] + + +_USER_TURN: Final = {"role": "user", "content": "hello"} + + +@pytest.mark.parametrize( + ("input_items", "param"), + [ + ([{"content": "hi"}], "input[0].role"), + ([{"role": None, "content": "hi"}], "input[0].role"), + ([{"role": "", "content": "hi"}], "input[0].role"), + ([{"type": "message", "content": "hi"}], "input[0].role"), + ([{"type": "message", "id": "msg_1"}], "input[0].role"), + ([{"content": [{"type": "input_text", "text": "hi"}]}], "input[0].role"), + ([{"content": None}], "input[0].role"), + ([{}], "input[0].role"), + ([{"id": "msg_1", "content": "hi"}], "input[0].role"), + ([_USER_TURN, {"content": "bye"}], "input[1].role"), + ([{"type": "additional_tools", "tools": []}, {"content": "hi"}], "input[1].role"), + ], +) +def test_input_item_without_role_answers_400_naming_it_and_sends_nothing(input_items, param): + forwarded: Final[list[object]] = [] + + with patch("litellm.completion", lambda **kwargs: forwarded.append(kwargs)): + with pytest.raises(litellm.BadRequestError) as raised: + LiteLLMCompletionTransformationHandler().response_api_handler( + model="claude-sonnet-5-5", + input=input_items, + responses_api_request={}, + custom_llm_provider="anthropic", + ) + + assert forwarded == [] + assert raised.value.status_code == 400 + assert raised.value.param == param + assert raised.value.type == "invalid_request_error" + assert raised.value.code == "missing_required_parameter" + assert param in str(raised.value) + assert raised.value.llm_provider == "anthropic" + + +@pytest.mark.parametrize( + "input_items", + [ + "hello", + [_USER_TURN], + [_USER_TURN, {"id": "msg_1"}], + [_USER_TURN, {"type": "item_reference", "id": "msg_1"}], + [_USER_TURN, {"type": "reasoning", "id": "rs_1", "summary": []}], + [ + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "done"}, + ], + [ResponseCompactionItem(id="cmp_1", encrypted_content="opaque", type="compaction"), _USER_TURN], + ], +) +def test_role_less_shapes_openai_accepts_still_reach_the_provider(input_items): + captured: Final[dict[str, object]] = {} + + def fake_completion(**kwargs): + captured.update(kwargs) + raise _StopForwarding() + + with patch("litellm.completion", fake_completion): + with pytest.raises(_StopForwarding): + LiteLLMCompletionTransformationHandler().response_api_handler( + model="claude-sonnet-5-5", + input=input_items, + responses_api_request={}, + custom_llm_provider="anthropic", + ) + + assert captured["messages"]