diff --git a/examples/bigmodel.md b/examples/bigmodel.md index 1ae64bcbfc4..1f0bb4cd2af 100644 --- a/examples/bigmodel.md +++ b/examples/bigmodel.md @@ -56,6 +56,8 @@ GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 suppo Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search +Codex sends `include` and `parallel_tool_calls` on every Responses request, and LiteLLM does not forward either to BigModel. Set `drop_params: true` on the deployment so LiteLLM drops them instead of answering 400 + The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 01baeed9bef..7f45c1eba28 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -268,6 +268,9 @@ class BaseResponsesAPIConfig(ABC): """Returns True if litellm should fake a stream for the given model and stream value""" return False + def may_answer_stream_with_json(self) -> bool: + return False + def supports_native_websocket(self) -> bool: """ Returns True if the provider has a native WebSocket endpoint for Responses API. diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py index 6812506729d..0957f57832c 100644 --- a/litellm/llms/bigmodel/responses/transformation.py +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -1,8 +1,9 @@ from collections.abc import Mapping +from types import MappingProxyType from typing import Final import httpx -from pydantic import JsonValue, TypeAdapter +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError from typing_extensions import Never from litellm.exceptions import AuthenticationError, UnsupportedParamsError @@ -43,8 +44,39 @@ _SUPPORTED_PARAMS: Final = ( ) +_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType( + { + **dict.fromkeys((1000, 1001, 1003, 1005), 401), + **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400), + 1220: 403, + **dict.fromkeys( + (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321), + 429, + ), + } +) + + +class _RequestedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + tools: JsonValue = None + tool_choice: JsonValue = None + + +def _requested_tools(logging_obj: Logging) -> _RequestedTools: + try: + return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {}) + except ValidationError: + return _RequestedTools() + + def _envelope_status_code(code: JsonValue) -> int: - return code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code <= 599 else 500 + if not isinstance(code, int) or isinstance(code, bool): + return 500 + if 400 <= code <= 599: + return code + return _STATUS_BY_BUSINESS_CODE.get(code, 500) def _normalize_output_item(item: JsonValue) -> JsonValue: @@ -59,6 +91,7 @@ def _normalize_output_item(item: JsonValue) -> JsonValue: def _normalize_response( body: Mapping[str, JsonValue], + requested: _RequestedTools, ) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary usage: Final = body.get("usage") output: Final = body.get("output") @@ -77,8 +110,8 @@ def _normalize_response( normalized: Final[JsonValue] = { "parallel_tool_calls": False, **body, - "tools": [] if tools is None else tools, - "tool_choice": "auto" if tool_choice is None else tool_choice, + "tools": (requested.tools or []) if tools is None else tools, + "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, **({"usage": normalized_usage} if "usage" in body else {}), **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), } @@ -115,6 +148,9 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): ) -> bool: return False + def may_answer_stream_with_json(self) -> bool: + return True + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list return list(_SUPPORTED_PARAMS) @@ -153,7 +189,7 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): status_code=_envelope_status_code(body.get("code")), headers=raw_response.headers, ) - normalized: Final = _normalize_response(body) + normalized: Final = _normalize_response(body, _requested_tools(logging_obj)) normalized_response: Final = httpx.Response( status_code=raw_response.status_code, headers={ @@ -179,7 +215,11 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): item: Final = chunk.get("item") normalized: Final[JsonValue] = { **chunk, - **({"response": _normalize_response(response)} if isinstance(response, dict) else {}), + **( + {"response": _normalize_response(response, _requested_tools(logging_obj))} + if isinstance(response, dict) + else {} + ), **({"item": _normalize_output_item(item)} if "item" in chunk else {}), } event_model: Final = self.get_event_model_class(str(normalized.get("type"))) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 0c74748ec7d..26a0d4df97f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3035,7 +3035,11 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True or (_is_sized_json_body(response) and _is_json_document(response.read())): + if fake_stream is True or ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(response.read()) + ): return MockResponsesAPIStreamingIterator( response=response, model=model, @@ -3239,7 +3243,9 @@ class BaseLLMHTTPHandler: if caller_requested_stream: if fake_stream is True or ( - _is_sized_json_body(response) and _is_json_document(await response.aread()) + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(await response.aread()) ): return MockResponsesAPIStreamingIterator( response=response, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index 5c485ef0291..a9ecbe66118 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -432,7 +432,12 @@ def test_openai_routes_bigmodel_does_not_serve_send_no_request( @pytest.mark.parametrize( ("code", "error_class", "status_code"), - ((401, litellm.AuthenticationError, 401), (1113, litellm.InternalServerError, 500)), + ( + (401, litellm.AuthenticationError, 401), + (1113, litellm.RateLimitError, 429), + (1301, litellm.BadRequestError, 400), + (1200, litellm.InternalServerError, 500), + ), ) @respx.mock def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( @@ -444,6 +449,27 @@ def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( assert getattr(raised.value, "status_code", None) == status_code +@pytest.mark.parametrize("stream", (False, True)) +@respx.mock +def test_responses_without_tools_in_the_answer_echo_the_requested_tools(stream: bool) -> None: + tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object", "properties": {}}}] + completed: Final = {"type": "response.completed", "sequence_number": 0, "response": response_payload()} + respx.post(f"{BASE}/api/v1/responses").respond( + **( + {"content": f"event: response.completed\ndata: {json.dumps(completed)}\n\n"} + if stream + else {"json": response_payload()} + ) + ) + answer: Final = litellm.responses( + model=f"bigmodel/{MODEL}", input="Hello", tools=tools, tool_choice="required", stream=stream + ) + response: Final = tuple(answer)[-1].response if stream else answer + validated: Final = OpenAIResponse.model_validate(response.model_dump()) + assert validated.tools[0].name == "weather" + assert validated.tool_choice == "required" + + @pytest.mark.asyncio @respx.mock async def test_responses_stream_answered_with_a_failure_envelope_raises_its_status() -> None: diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 297e64629a8..2590567726e 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5695,3 +5695,16 @@ async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( await client.close() else: client.client.close() + + +@pytest.mark.asyncio +async def test_a_stream_answered_with_one_json_body_is_not_replayed_for_a_provider_that_does_not_opt_in( + httpx_transport: None, +) -> None: + with respx.mock(assert_all_called=True) as router: + router.post("https://api.openai.com/v1/responses").respond(json={"error": {"message": "quota exceeded"}}) + stream: Final = await litellm.aresponses( + model="openai/gpt-5.5", input="Hello", api_key="fake-key", stream=True + ) + with pytest.raises(litellm.exceptions.MidStreamFallbackError, match="before any terminal event"): + _ = tuple([event async for event in stream])