scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-08 08:08:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix(chatgpt): forward text and max_output_tokens to the Codex backend (#38086)

Browse source 
* fix(chatgpt): keep text and accept string input for the Codex backend

Two request-side defects stop chatgpt/* serving structured output or a string
input.

The allowlist at the end of transform_responses_api_request omits "text". The
chat-to-responses bridge translates response_format into text.format, so the
allowlist discards strict schemas silently and the backend answers with prose
instead of JSON. The backend does honour text.format when it receives it.

Separately the backend rejects a bare string input with
{"detail": "Input must be a list"}, while the Responses API itself accepts
either shape, so the string needs wrapping before it is sent.

Both are request-shaping only, so neither depends on the streaming work in
#31332 or #34095, though those two block the same call and have to be in place
to observe this one end to end.

The "text" case was reported in #24356 and closed by the stale bot without a
fix.

Signed-off-by: Alexander Chernov <alexander@chernov.it>

* fix(chatgpt): annotate the input coercion for the type-discipline gate

The LIT002 budget counts mutable-collection construction, and wrapping the
string input builds a list and a dict, which put the total two over its
ceiling. Both constructions are the wire payload this backend requires, so the
list cannot be replaced with an immutable equivalent; annotate the line with a
reason instead, matching the convention used elsewhere in the tree.

LIT005 freezes reasonless suppressions at zero, so the reason is required
rather than decorative.

Signed-off-by: Alexander Chernov <alexander@chernov.it>

* refactor(chatgpt): bind the coerced input instead of rebinding the parameter

Greptile flagged the parameter rebinding and the explanatory comments against
the repo conventions in CLAUDE.md, which bans rebinding a function parameter
(LIT011) and keeps comments to suppressions, TODOs and genuinely complex logic

Bind a Final local for the coerced input rather than reassigning `input`, and
drop both prose comments; the rationale lives in the commit and the PR body
rather than duplicated at the call site. The mutable-ok suppression stays,
since it is the allowed kind and the list is still constructed

Behaviour is unchanged: 24 tests pass, and the same 4 fail with only the
provider file reverted to base

Signed-off-by: Alexander Chernov <alexander@chernov.it>

* fix(chatgpt): forward max_output_tokens to the Codex backend and return incomplete non-stream responses

The Codex backend accepts and enforces max_output_tokens, so the allowlist now keeps it
instead of dropping it. A structured-output request could otherwise stream until the
backend gave up, ignoring the caller's max_tokens.

A capped request ends with response.incomplete, so the non-stream SSE aggregator now
treats that event as terminal and returns the partial response instead of an error.

---------

Signed-off-by: Alexander Chernov <alexander@chernov.it>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Alexander Chernov 2026-11-06 14:14:32 +00:00 • committed by GitHub
parent febbe38326
commit d1cde1002e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 117 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/llms/chatgpt/responses/transformation.py
View file

@ -143,6 +143,8 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
"reasoning",
"previous_response_id",
"truncation",
"text",
"max_output_tokens",
}
filtered: Final = {k: v for k, v in request.items() if k in allowed_keys}
@ -218,7 +220,10 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
)
continue
if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
if event_type in (
ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
):
# Real OUTPUT_ITEM_DONE events take precedence at any given
# output_index, but text-only items at indices without a
# matching OUTPUT_ITEM_DONE must still be preserved (e.g.

3
tests/integration/_support/codex_vendor.py
View file

@ -46,6 +46,9 @@ FORWARDED_KEYS: Final = frozenset(
"reasoning",
"previous_response_id",
"truncation",
"text",
"service_tier",
"max_output_tokens",
}
)

120
tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py
View file

@ -19,6 +19,7 @@ from litellm.llms.chatgpt.authenticator import Authenticator
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig
from litellm.llms.openai.common_utils import OpenAIError
from litellm.llms.openai.responses.transformation import OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS
from litellm.main import responses_api_bridge_check
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
@ -59,7 +60,7 @@ class TestChatGPTResponsesAPITransformation:
assert request["reasoning"] == {"effort": effort}
assert request["stream"] is True
assert request["store"] is False
assert "max_output_tokens" not in request
assert request["max_output_tokens"] == 16
assert "prompt_cache_options" not in request
@pytest.mark.parametrize("requested_tier", [None, "auto", "flex", "unknown"])
@ -236,9 +237,9 @@ class TestChatGPTResponsesAPITransformation:
"top_p": 0.9,
"context_management": [{"type": "compaction", "compact_threshold": 200000}],
"metadata": {"foo": "bar"},
"max_output_tokens": 123,
"stream_options": {"include_usage": True},
# supported and should be preserved
"max_output_tokens": 123,
"truncation": "auto",
"previous_response_id": "resp_123",
"reasoning": {"effort": "medium"},
@ -254,9 +255,9 @@ class TestChatGPTResponsesAPITransformation:
assert "top_p" not in request
assert "context_management" not in request
assert "metadata" not in request
assert "max_output_tokens" not in request
assert "stream_options" not in request
assert request["max_output_tokens"] == 123
assert request["truncation"] == "auto"
assert request["previous_response_id"] == "resp_123"
assert request["reasoning"] == {"effort": "medium"}
@ -618,22 +619,25 @@ def test_chatgpt_non_stream_sse_keeps_an_output_item_whose_text_holds_unicode_li
assert parsed.output_text == text
@pytest.mark.parametrize("call", ["responses", "completion"])
def test_prompt_cache_key_keeps_one_session_header_across_requests(
call: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
@pytest.fixture
def chatgpt_token_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
(tmp_path / "auth.json").write_text(
json.dumps({"access_token": "token", "account_id": "acct", "expires_at": 4102444800})
)
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path))
monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False)
def _completed_codex_upstream(output_text: str, incomplete_reason: str | None = None) -> MagicMock:
status: Final = "completed" if incomplete_reason is None else "incomplete"
completed: Final = {
"type": "response.completed",
"type": f"response.{status}",
"response": {
"id": "resp_1",
"object": "response",
"created_at": 1,
"status": "completed",
"status": status,
"incomplete_details": None if incomplete_reason is None else {"reason": incomplete_reason},
"model": "gpt-5.5",
"output": [
{
@ -641,19 +645,25 @@ def test_prompt_cache_key_keeps_one_session_header_across_requests(
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "OK", "annotations": []}],
"content": [{"type": "output_text", "text": output_text, "annotations": []}],
}
],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
}
upstream: Final = MagicMock(
return MagicMock(
side_effect=lambda _request: httpx.Response(
200,
headers={"content-type": "text/event-stream"},
content=f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode(),
content=f"event: response.{status}\ndata: {json.dumps(completed)}\n\n".encode(),
)
)
@pytest.mark.usefixtures("chatgpt_token_dir")
@pytest.mark.parametrize("call", ["responses", "completion"])
def test_prompt_cache_key_keeps_one_session_header_across_requests(call: str) -> None:
upstream: Final = _completed_codex_upstream("OK")
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream)))
for _ in range(2):
if call == "responses":
@ -667,3 +677,89 @@ def test_prompt_cache_key_keeps_one_session_header_across_requests(
)
assert [call.args[0].headers["session_id"] for call in upstream.call_args_list] == ["conv-123", "conv-123"]
_CITY_SCHEMA: Final = {
"type": "object",
"properties": {"city": {"type": "string"}, "country": {"type": "string"}},
"required": ["city", "country"],
"additionalProperties": False,
}
@pytest.mark.usefixtures("chatgpt_token_dir")
def test_responses_text_reaches_the_codex_backend_whole() -> None:
text: Final = {
"format": {"type": "json_schema", "name": "city", "schema": _CITY_SCHEMA, "strict": True},
"verbosity": "low",
}
upstream: Final = _completed_codex_upstream('{"city": "Paris", "country": "France"}')
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream)))
litellm.responses(model="chatgpt/gpt-5.5", input="Where is the Eiffel Tower?", text=text, client=client)
assert json.loads(upstream.call_args.args[0].content)["text"] == text
@pytest.mark.usefixtures("chatgpt_token_dir")
@pytest.mark.parametrize(
("response_format", "expected_format"),
[
(
{"type": "json_schema", "json_schema": {"name": "city", "schema": _CITY_SCHEMA, "strict": True}},
{"type": "json_schema", "name": "city", "schema": _CITY_SCHEMA, "strict": True},
),
({"type": "json_object"}, {"type": "json_object"}),
],
)
def test_chat_response_format_reaches_the_codex_backend_as_text_format(
response_format: dict[str, object], expected_format: dict[str, object]
) -> None:
upstream: Final = _completed_codex_upstream('{"city": "Paris", "country": "France"}')
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream)))
litellm.completion(
model="chatgpt/gpt-5.5",
messages=[{"role": "user", "content": "Where is the Eiffel Tower? Answer in JSON."}],
response_format=response_format,
client=client,
)
assert json.loads(upstream.call_args.args[0].content)["text"]["format"] == expected_format
@pytest.mark.usefixtures("chatgpt_token_dir")
@pytest.mark.parametrize(
("max_tokens", "expected_max_output_tokens"),
[(300, 300), (OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS - 1, OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS)],
)
def test_chat_max_tokens_reaches_the_codex_backend_no_lower_than_its_minimum(
max_tokens: int, expected_max_output_tokens: int
) -> None:
upstream: Final = _completed_codex_upstream("OK")
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream)))
litellm.completion(
model="chatgpt/gpt-5.5",
messages=[{"role": "user", "content": "hi"}],
max_tokens=max_tokens,
client=client,
)
assert json.loads(upstream.call_args.args[0].content)["max_output_tokens"] == expected_max_output_tokens
@pytest.mark.usefixtures("chatgpt_token_dir")
def test_non_stream_chat_cut_off_by_max_tokens_returns_the_partial_answer() -> None:
upstream: Final = _completed_codex_upstream('{"capitals": ["Paris", "Ber', incomplete_reason="max_output_tokens")
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(upstream)))
response: Final = litellm.completion(
model="chatgpt/gpt-5.5",
messages=[{"role": "user", "content": "List capitals as JSON."}],
max_tokens=16,
client=client,
)
assert response.choices[0].finish_reason == "length"
assert response.choices[0].message.content == '{"capitals": ["Paris", "Ber'