From d1cde1002e374003490f3e12a50a77174f4841de Mon Sep 17 00:00:00 2001 From: Alexander Chernov Date: Fri, 6 Nov 2026 14:14:32 +0000 Subject: [PATCH] fix(chatgpt): forward text and max_output_tokens to the Codex backend (#38086) * fix(chatgpt): keep text and accept string input for the Codex backend Two request-side defects stop chatgpt/* serving structured output or a string input. The allowlist at the end of transform_responses_api_request omits "text". The chat-to-responses bridge translates response_format into text.format, so the allowlist discards strict schemas silently and the backend answers with prose instead of JSON. The backend does honour text.format when it receives it. Separately the backend rejects a bare string input with {"detail": "Input must be a list"}, while the Responses API itself accepts either shape, so the string needs wrapping before it is sent. Both are request-shaping only, so neither depends on the streaming work in #31332 or #34095, though those two block the same call and have to be in place to observe this one end to end. The "text" case was reported in #24356 and closed by the stale bot without a fix. Signed-off-by: Alexander Chernov * fix(chatgpt): annotate the input coercion for the type-discipline gate The LIT002 budget counts mutable-collection construction, and wrapping the string input builds a list and a dict, which put the total two over its ceiling. Both constructions are the wire payload this backend requires, so the list cannot be replaced with an immutable equivalent; annotate the line with a reason instead, matching the convention used elsewhere in the tree. LIT005 freezes reasonless suppressions at zero, so the reason is required rather than decorative. Signed-off-by: Alexander Chernov * refactor(chatgpt): bind the coerced input instead of rebinding the parameter Greptile flagged the parameter rebinding and the explanatory comments against the repo conventions in CLAUDE.md, which bans rebinding a function parameter (LIT011) and keeps comments to suppressions, TODOs and genuinely complex logic Bind a Final local for the coerced input rather than reassigning `input`, and drop both prose comments; the rationale lives in the commit and the PR body rather than duplicated at the call site. The mutable-ok suppression stays, since it is the allowed kind and the list is still constructed Behaviour is unchanged: 24 tests pass, and the same 4 fail with only the provider file reverted to base Signed-off-by: Alexander Chernov * fix(chatgpt): forward max_output_tokens to the Codex backend and return incomplete non-stream responses The Codex backend accepts and enforces max_output_tokens, so the allowlist now keeps it instead of dropping it. A structured-output request could otherwise stream until the backend gave up, ignoring the caller's max_tokens. A capped request ends with response.incomplete, so the non-stream SSE aggregator now treats that event as terminal and returns the partial response instead of an error. --------- Signed-off-by: Alexander Chernov Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/chatgpt/responses/transformation.py | 7 +- tests/integration/_support/codex_vendor.py | 3 + .../test_chatgpt_responses_transformation.py | 120 ++++++++++++++++-- 3 files changed, 117 insertions(+), 13 deletions(-) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 3c7d0b8d673..638a55452d2 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -143,6 +143,8 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): "reasoning", "previous_response_id", "truncation", + "text", + "max_output_tokens", } filtered: Final = {k: v for k, v in request.items() if k in allowed_keys} @@ -218,7 +220,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): ) continue - if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + if event_type in ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ): # Real OUTPUT_ITEM_DONE events take precedence at any given # output_index, but text-only items at indices without a # matching OUTPUT_ITEM_DONE must still be preserved (e.g. diff --git a/tests/integration/_support/codex_vendor.py b/tests/integration/_support/codex_vendor.py index 2ec9c4e8b96..a588fa8db3d 100644 --- a/tests/integration/_support/codex_vendor.py +++ b/tests/integration/_support/codex_vendor.py @@ -46,6 +46,9 @@ FORWARDED_KEYS: Final = frozenset( "reasoning", "previous_response_id", "truncation", + "text", + "service_tier", + "max_output_tokens", } ) diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 635363e77ea..c456e782c09 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -19,6 +19,7 @@ from litellm.llms.chatgpt.authenticator import Authenticator from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.openai.responses.transformation import OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS from litellm.main import responses_api_bridge_check from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -59,7 +60,7 @@ class TestChatGPTResponsesAPITransformation: assert request["reasoning"] == {"effort": effort} assert request["stream"] is True assert request["store"] is False - assert "max_output_tokens" not in request + assert request["max_output_tokens"] == 16 assert "prompt_cache_options" not in request @pytest.mark.parametrize("requested_tier", [None, "auto", "flex", "unknown"]) @@ -236,9 +237,9 @@ class TestChatGPTResponsesAPITransformation: "top_p": 0.9, "context_management": [{"type": "compaction", "compact_threshold": 200000}], "metadata": {"foo": "bar"}, - "max_output_tokens": 123, "stream_options": {"include_usage": True}, # supported and should be preserved + "max_output_tokens": 123, "truncation": "auto", "previous_response_id": "resp_123", "reasoning": {"effort": "medium"}, @@ -254,9 +255,9 @@ class TestChatGPTResponsesAPITransformation: assert "top_p" not in request assert "context_management" not in request assert "metadata" not in request - assert "max_output_tokens" not in request assert "stream_options" not in request + assert request["max_output_tokens"] == 123 assert request["truncation"] == "auto" assert request["previous_response_id"] == "resp_123" assert request["reasoning"] == {"effort": "medium"} @@ -618,22 +619,25 @@ def test_chatgpt_non_stream_sse_keeps_an_output_item_whose_text_holds_unicode_li assert parsed.output_text == text -@pytest.mark.parametrize("call", ["responses", "completion"]) -def test_prompt_cache_key_keeps_one_session_header_across_requests( - call: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: +@pytest.fixture +def chatgpt_token_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: (tmp_path / "auth.json").write_text( json.dumps({"access_token": "token", "account_id": "acct", "expires_at": 4102444800}) ) monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + + +def _completed_codex_upstream(output_text: str, incomplete_reason: str | None = None) -> MagicMock: + status: Final = "completed" if incomplete_reason is None else "incomplete" completed: Final = { - "type": "response.completed", + "type": f"response.{status}", "response": { "id": "resp_1", "object": "response", "created_at": 1, - "status": "completed", + "status": status, + "incomplete_details": None if incomplete_reason is None else {"reason": incomplete_reason}, "model": "gpt-5.5", "output": [ { @@ -641,19 +645,25 @@ def test_prompt_cache_key_keeps_one_session_header_across_requests( "id": "msg_1", "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "OK", "annotations": []}], + "content": [{"type": "output_text", "text": output_text, "annotations": []}], } ], "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, }, } - upstream: Final = MagicMock( + return MagicMock( side_effect=lambda _request: httpx.Response( 200, headers={"content-type": "text/event-stream"}, - content=f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode(), + content=f"event: response.{status}\ndata: {json.dumps(completed)}\n\n".encode(), ) ) + + +@pytest.mark.usefixtures("chatgpt_token_dir") +@pytest.mark.parametrize("call", ["responses", "completion"]) +def test_prompt_cache_key_keeps_one_session_header_across_requests(call: str) -> None: + upstream: Final = _completed_codex_upstream("OK") client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream))) for _ in range(2): if call == "responses": @@ -667,3 +677,89 @@ def test_prompt_cache_key_keeps_one_session_header_across_requests( ) assert [call.args[0].headers["session_id"] for call in upstream.call_args_list] == ["conv-123", "conv-123"] + + +_CITY_SCHEMA: Final = { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": False, +} + + +@pytest.mark.usefixtures("chatgpt_token_dir") +def test_responses_text_reaches_the_codex_backend_whole() -> None: + text: Final = { + "format": {"type": "json_schema", "name": "city", "schema": _CITY_SCHEMA, "strict": True}, + "verbosity": "low", + } + upstream: Final = _completed_codex_upstream('{"city": "Paris", "country": "France"}') + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream))) + + litellm.responses(model="chatgpt/gpt-5.5", input="Where is the Eiffel Tower?", text=text, client=client) + + assert json.loads(upstream.call_args.args[0].content)["text"] == text + + +@pytest.mark.usefixtures("chatgpt_token_dir") +@pytest.mark.parametrize( + ("response_format", "expected_format"), + [ + ( + {"type": "json_schema", "json_schema": {"name": "city", "schema": _CITY_SCHEMA, "strict": True}}, + {"type": "json_schema", "name": "city", "schema": _CITY_SCHEMA, "strict": True}, + ), + ({"type": "json_object"}, {"type": "json_object"}), + ], +) +def test_chat_response_format_reaches_the_codex_backend_as_text_format( + response_format: dict[str, object], expected_format: dict[str, object] +) -> None: + upstream: Final = _completed_codex_upstream('{"city": "Paris", "country": "France"}') + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream))) + + litellm.completion( + model="chatgpt/gpt-5.5", + messages=[{"role": "user", "content": "Where is the Eiffel Tower? Answer in JSON."}], + response_format=response_format, + client=client, + ) + + assert json.loads(upstream.call_args.args[0].content)["text"]["format"] == expected_format + + +@pytest.mark.usefixtures("chatgpt_token_dir") +@pytest.mark.parametrize( + ("max_tokens", "expected_max_output_tokens"), + [(300, 300), (OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS - 1, OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS)], +) +def test_chat_max_tokens_reaches_the_codex_backend_no_lower_than_its_minimum( + max_tokens: int, expected_max_output_tokens: int +) -> None: + upstream: Final = _completed_codex_upstream("OK") + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream))) + + litellm.completion( + model="chatgpt/gpt-5.5", + messages=[{"role": "user", "content": "hi"}], + max_tokens=max_tokens, + client=client, + ) + + assert json.loads(upstream.call_args.args[0].content)["max_output_tokens"] == expected_max_output_tokens + + +@pytest.mark.usefixtures("chatgpt_token_dir") +def test_non_stream_chat_cut_off_by_max_tokens_returns_the_partial_answer() -> None: + upstream: Final = _completed_codex_upstream('{"capitals": ["Paris", "Ber', incomplete_reason="max_output_tokens") + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream))) + + response: Final = litellm.completion( + model="chatgpt/gpt-5.5", + messages=[{"role": "user", "content": "List capitals as JSON."}], + max_tokens=16, + client=client, + ) + + assert response.choices[0].finish_reason == "length" + assert response.choices[0].message.content == '{"capitals": ["Paris", "Ber'