Merge pull request #41939 from BerriAI/litellm_responses_missing_input_400

* fix(proxy): return 400 instead of 500 for /v1/responses without input

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): validate input before starting background responses polling

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): validate responses input after prompt template expansion

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yucheng <yucheng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/response_api_endpoints/endpoints.py

@@ -34,6 +34,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
     _read_request_body,
     _safe_set_request_parsed_body,
 )
+from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
 from litellm.types.llms.openai import (
     REASONING_EFFORT,
     ResponsesAPIOptionalRequestParams,
@@ -296,6 +297,7 @@ async def responses_api(
                 route_type="aresponses",
                 llm_router=llm_router,
             )
+            raise_if_required_body_param_missing(route_type="aresponses", data=data)
         except Exception as e:
             raise await processor._handle_llm_api_exception(
                 e=e,

litellm/proxy/route_llm_request.py

@@ -159,6 +159,7 @@ class ProxyModelNotFoundError(HTTPException):
 REQUIRED_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = {
     "acompletion": ("messages",),
     "aembedding": ("input",),
+    "aresponses": ("input",),
     "acreate_batch": ("input_file_id", "endpoint", "completion_window"),
 }
 

tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py

@@ -121,6 +121,137 @@ async def test_streaming_upstream_errors_keep_the_client_protocol(
         assert "error" in events[-1]
 
 
+@pytest.mark.asyncio
+async def test_responses_api_background_polling_rejects_missing_input():
+    from fastapi import Response as FastAPIResponse
+    from starlette.requests import Request
+
+    from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+    from litellm.proxy.response_api_endpoints.endpoints import responses_api
+
+    processor = MagicMock()
+
+    async def return_exception(*, e: Exception, **kwargs: object) -> Exception:
+        return e
+
+    processor._handle_llm_api_exception = AsyncMock(side_effect=return_exception)
+    processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o"}, MagicMock()))
+
+    async def receive():
+        return {
+            "type": "http.request",
+            "body": b'{"model":"gpt-4o","background":true}',
+            "more_body": False,
+        }
+
+    request = Request(
+        {
+            "type": "http",
+            "method": "POST",
+            "path": "/v1/responses",
+            "headers": [(b"content-type", b"application/json")],
+        },
+        receive,
+    )
+
+    with (
+        patch(  # test-quality-ok: endpoint constructs the processor directly
+            "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
+            return_value=processor,
+        ),
+        patch(  # test-quality-ok: polling decision is imported inside the endpoint
+            "litellm.proxy.response_polling.polling_handler.should_use_polling_for_request",
+            return_value=True,
+        ),
+        patch(  # test-quality-ok: background task is imported inside the endpoint
+            "litellm.proxy.response_polling.background_streaming.background_streaming_task",
+            new_callable=AsyncMock,
+        ) as mock_background_streaming_task,
+        patch(  # test-quality-ok: polling handler is imported inside the endpoint
+            "litellm.proxy.response_polling.polling_handler.ResponsePollingHandler.create_initial_state",
+            new_callable=AsyncMock,
+        ) as mock_create_initial_state,
+    ):
+        with pytest.raises(ProxyException) as exc_info:
+            await responses_api(
+                request=request,
+                fastapi_response=FastAPIResponse(),
+                user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+            )
+
+    assert exc_info.value.code == "400"
+    assert exc_info.value.param == "input"
+    processor.common_processing_pre_call_logic.assert_awaited_once()
+    mock_background_streaming_task.assert_not_called()
+    mock_create_initial_state.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_responses_api_background_polling_accepts_input_from_prompt_template():
+    from fastapi import Response as FastAPIResponse
+    from starlette.requests import Request
+
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.response_api_endpoints.endpoints import responses_api
+
+    processor = MagicMock()
+    processor.common_processing_pre_call_logic = AsyncMock(
+        return_value=({"model": "gpt-4o", "input": "hello from prompt"}, MagicMock())
+    )
+    initial_state = MagicMock()
+
+    async def receive():
+        return {
+            "type": "http.request",
+            "body": b'{"model":"gpt-4o","prompt_id":"greeting","background":true}',
+            "more_body": False,
+        }
+
+    request = Request(
+        {
+            "type": "http",
+            "method": "POST",
+            "path": "/v1/responses",
+            "headers": [(b"content-type", b"application/json")],
+        },
+        receive,
+    )
+
+    with (
+        patch(  # test-quality-ok: endpoint constructs the processor directly
+            "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing",
+            return_value=processor,
+        ),
+        patch(  # test-quality-ok: polling decision is imported inside the endpoint
+            "litellm.proxy.response_polling.polling_handler.should_use_polling_for_request",
+            return_value=True,
+        ),
+        patch(  # test-quality-ok: background task is imported inside the endpoint
+            "litellm.proxy.response_polling.background_streaming.background_streaming_task",
+            new_callable=AsyncMock,
+        ),
+        patch(  # test-quality-ok: avoid scheduling a background task in this unit test
+            "litellm.proxy.response_api_endpoints.endpoints.asyncio.create_task",
+        ),
+        patch(  # test-quality-ok: polling handler is imported inside the endpoint
+            "litellm.proxy.response_polling.polling_handler.ResponsePollingHandler.create_initial_state",
+            new_callable=AsyncMock,
+        ) as mock_create_initial_state,
+    ):
+        mock_create_initial_state.return_value = initial_state
+        result = await responses_api(
+            request=request,
+            fastapi_response=FastAPIResponse(),
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+        )
+
+    assert result is initial_state
+    processor.common_processing_pre_call_logic.assert_awaited_once()
+    mock_create_initial_state.assert_awaited_once()
+    request_data = mock_create_initial_state.await_args.kwargs["request_data"]
+    assert request_data["input"] == "hello from prompt"
+
+
 class TestResponsesAPIEndpoints(unittest.TestCase):
     @pytest.mark.asyncio
     @patch("litellm.proxy.proxy_server.llm_router")

tests/test_litellm/proxy/test_route_llm_request.py

@@ -1043,6 +1043,7 @@ async def test_route_request_override_enable_tag_filtering_beats_body_value():
     [
         ("acompletion", "messages", "/chat/completions"),
         ("aembedding", "input", "/embeddings"),
+        ("aresponses", "input", "/responses"),
         ("acreate_batch", "input_file_id", "/batches"),
     ],
 )
@@ -1090,6 +1091,8 @@ def test_raise_if_required_body_param_missing_names_first_missing_batch_param(da
         ("acompletion", {"model": "gpt-4o", "messages": []}),
         ("atext_completion", {"model": "gpt-4o"}),
         ("aembedding", {"model": "text-embedding-3-small", "input": "hi"}),
+        ("aresponses", {"model": "gpt-4o", "input": "hi"}),
+        ("aresponses", {"model": "gpt-4o", "input": []}),
         ("arerank", {"model": "rerank-model"}),
         ("aimage_generation", {"model": "dall-e-3"}),
         (
@@ -1120,6 +1123,20 @@ async def test_route_request_rejects_chat_completion_without_messages():
     llm_router.acompletion.assert_not_called()
 
 
+@pytest.mark.asyncio
+async def test_route_request_rejects_responses_without_input():
+    from litellm.proxy.route_llm_request import ProxyMissingRequiredParamError
+
+    llm_router = MagicMock()
+
+    with pytest.raises(ProxyMissingRequiredParamError) as exc_info:
+        await route_request({"model": "gpt-4o"}, llm_router, None, "aresponses")
+
+    assert exc_info.value.code == "400"
+    assert exc_info.value.param == "input"
+    llm_router.aresponses.assert_not_called()
+
+
 class FakeProxyModelTable:
     def __init__(self, rows):
         self.rows = rows