From 2bd6cc46b3572f6c4e479c375c308a2a6931c773 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:20:43 -0700 Subject: [PATCH] fix(bigmodel): keep the Coding Plan base, surface Responses auth errors, and scope routes Chat Completions keeps a GLM Coding Plan api_base. Responses raises the status from BigModel's success=false body instead of returning an empty 200, and a JSON body answering a stream request is replayed as a one-shot stream. Messages streams keep ping and [DONE] lines. bigmodel leaves openai_compatible_providers so speech, image, and transcription routes send it no request --- litellm/constants.py | 1 - litellm/llms/bigmodel/common_utils.py | 12 ++- .../llms/bigmodel/messages/transformation.py | 22 +++- .../llms/bigmodel/responses/transformation.py | 14 ++- litellm/llms/custom_httpx/llm_http_handler.py | 19 +++- litellm/main.py | 1 + litellm/utils.py | 1 + .../llms/bigmodel/test_bigmodel_provider.py | 102 ++++++++++++++++-- 8 files changed, 151 insertions(+), 21 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 84ee4a3fca9..8cc51d0ab5e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1084,7 +1084,6 @@ openai_compatible_providers: Final[list[str]] = [ "xinference", "xai", "zai", - "bigmodel", "together_ai", "fireworks_ai", "empower", diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py index d17f478ed29..deb9e54d78b 100644 --- a/litellm/llms/bigmodel/common_utils.py +++ b/litellm/llms/bigmodel/common_utils.py @@ -5,7 +5,10 @@ from urllib.parse import urlsplit, urlunsplit from litellm.secret_managers.main import get_secret_str BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_CODING_CHAT_PATH: Final = "/api/coding/paas/v4" _API_PATHS: Final = ( + "/api/coding/paas/v4/chat/completions", + _CODING_CHAT_PATH, "/api/paas/v4/chat/completions", "/api/anthropic/v1/messages", "/api/anthropic/v1", @@ -32,7 +35,10 @@ def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses" resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE parsed: Final = urlsplit(resolved_base) path: Final = parsed.path.rstrip("/") - root: Final = next((path.removesuffix(suffix) for suffix in _API_PATHS if path.endswith(suffix)), path) - return urlunsplit( - (parsed.scheme, parsed.netloc, f"{root}{_ENDPOINT_PATHS[endpoint]}", parsed.query, parsed.fragment) + suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "") + endpoint_path: Final = ( + _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint] + ) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment) ) diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py index 2cac2d3b899..041adf49374 100644 --- a/litellm/llms/bigmodel/messages/transformation.py +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -4,7 +4,7 @@ from typing import Final import httpx from openai._streaming import ServerSentEvent, SSEDecoder -from pydantic import JsonValue, TypeAdapter +from pydantic import JsonValue, TypeAdapter, ValidationError from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging @@ -45,20 +45,34 @@ class _BigModelMessagesStream(httpx.AsyncByteStream): await self._response.aclose() -def _normalize_event(event: ServerSentEvent) -> bytes: - body: Final = _JSON_OBJECT.validate_json(event.data) +def _json_object(data: str) -> Mapping[str, JsonValue] | None: + try: + return _JSON_OBJECT.validate_json(data) + except ValidationError: + return None + + +def _normalized_data(data: str) -> str: + body: Final = _json_object(data) + if body is None: + return data message: Final = body.get("message") normalized: Final[JsonValue] = ( {**body, "message": _normalize_message(message)} if body.get("type") == "message_start" and isinstance(message, dict) else _normalize_message(body) ) + return json.dumps(normalized) + + +def _normalize_event(event: ServerSentEvent) -> bytes: metadata: Final = "".join( f"{name}: {value}\n" for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) if value is not None ) - return f"{metadata}data: {json.dumps(normalized)}\n\n".encode() + data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n")) + return f"{metadata}{data_lines}\n".encode() class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py index a61f006371e..6812506729d 100644 --- a/litellm/llms/bigmodel/responses/transformation.py +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -7,6 +7,7 @@ from typing_extensions import Never from litellm.exceptions import AuthenticationError, UnsupportedParamsError from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ( ResponseInputParam, @@ -42,6 +43,10 @@ _SUPPORTED_PARAMS: Final = ( ) +def _envelope_status_code(code: JsonValue) -> int: + return code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code <= 599 else 500 + + def _normalize_output_item(item: JsonValue) -> JsonValue: if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): return item @@ -141,7 +146,14 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_response_api_response( self, model: str, raw_response: httpx.Response, logging_obj: Logging ) -> ResponsesAPIResponse: - normalized: Final = _normalize_response(_JSON_OBJECT.validate_json(raw_response.content)) + body: Final = _JSON_OBJECT.validate_json(raw_response.content) + if body.get("success") is False: + raise BaseLLMException( + message=str(body.get("msg") or raw_response.text), + status_code=_envelope_status_code(body.get("code")), + headers=raw_response.headers, + ) + normalized: Final = _normalize_response(body) normalized_response: Final = httpx.Response( status_code=raw_response.status_code, headers={ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ef009e70137..afc71f681dd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -365,6 +365,19 @@ class _SignedBodyKwargs(TypedDict, total=False): json: ReadOnly[dict[str, object]] +def _is_json_content(response: httpx.Response) -> bool: + content_types: Final = httpx.Headers(response.headers).get_list("content-type") + media_types: Final = (value.split(";")[0].strip().lower() for value in content_types) + return "application/json" in media_types + + +def _is_json_document(content: bytes) -> bool: + try: + return isinstance(json.loads(content), dict) + except ValueError: + return False + + def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs: if signed_body is not None: return {"data": signed_body} @@ -3022,7 +3035,7 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True: + if fake_stream is True or (_is_json_content(response) and _is_json_document(response.read())): return MockResponsesAPIStreamingIterator( response=response, model=model, @@ -3225,7 +3238,9 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: - if fake_stream is True: + if fake_stream is True or ( + _is_json_content(response) and _is_json_document(await response.aread()) + ): return MockResponsesAPIStreamingIterator( response=response, model=model, diff --git a/litellm/main.py b/litellm/main.py index 57d49c43add..d5427e8062a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6280,6 +6280,7 @@ def completion( or custom_llm_provider == "nebius" or custom_llm_provider == "wandb" or custom_llm_provider == "clarifai" + or custom_llm_provider == "bigmodel" or custom_llm_provider in litellm.openai_compatible_providers or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo diff --git a/litellm/utils.py b/litellm/utils.py index 91922210292..a232fd2f96a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5641,6 +5641,7 @@ def add_provider_specific_params_to_optional_params( "openai", "azure", "text-completion-openai", + "bigmodel", *litellm.openai_compatible_providers, ] or JSONProviderRegistry.exists(custom_llm_provider) if calls_through_openai_sdk: diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index f53673830b2..bb48827f535 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -1,6 +1,8 @@ +import contextlib import gzip +import io import json -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from functools import partial from itertools import chain from typing import Final, Literal @@ -100,7 +102,6 @@ def test_provider_resolution_and_all_native_endpoints() -> None: ProviderConfigManager.get_provider_anthropic_messages_config(MODEL, LlmProviders.BIGMODEL), BigModelMessagesConfig, ) - assert "bigmodel" in litellm.openai_compatible_providers assert "bigmodel" in litellm.provider_list assert litellm.get_llm_provider(f"zai/{MODEL}")[3] == "https://api.z.ai/api/paas/v4" @@ -112,6 +113,15 @@ def test_one_base_selects_each_api_without_duplicate_segments(path: str) -> None assert get_bigmodel_url(f"{BASE}{path}", "messages") == f"{BASE}/api/anthropic/v1/messages" +@pytest.mark.parametrize( + "path", ("/api/coding/paas/v4", "/api/coding/paas/v4/", "/api/coding/paas/v4/chat/completions") +) +def test_coding_plan_base_keeps_its_chat_path_and_shares_the_other_apis(path: str) -> None: + assert get_bigmodel_url(f"{BASE}{path}", "chat") == f"{BASE}/api/coding/paas/v4" + assert get_bigmodel_url(f"{BASE}{path}", "responses") == f"{BASE}/api/v1/responses" + assert get_bigmodel_url(f"{BASE}{path}", "messages") == f"{BASE}/api/anthropic/v1/messages" + + def test_custom_base_and_environment_precedence(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("BIGMODEL_API_BASE", "https://env.example/prefix/api/v1") assert get_bigmodel_url(None, "messages") == "https://env.example/prefix/api/anthropic/v1/messages" @@ -295,10 +305,11 @@ async def test_messages_forward_native_tools_thinking_and_usage(web_fetch_reques (("response.completed", "completed"), ("response.incomplete", "incomplete")), ) @pytest.mark.parametrize("include_logprobs", (False, True)) +@pytest.mark.parametrize("content_type", ("text/event-stream", "application/json")) @pytest.mark.asyncio @respx.mock async def test_responses_sse_without_done_preserves_terminal_usage( - terminal_type: str, status: str, include_logprobs: bool + terminal_type: str, status: str, include_logprobs: bool, content_type: str ) -> None: logprobs: Final = [{"token": "Hello", "bytes": [72, 101, 108, 108, 111], "logprob": -0.1, "top_logprobs": []}] events: Final = ( @@ -348,7 +359,7 @@ async def test_responses_sse_without_done_preserves_terminal_usage( {"type": terminal_type, "sequence_number": 5, "response": {**response_payload(), "status": status}}, ) sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) - respx.post(f"{BASE}/api/v1/responses").respond(content=sse, headers={"content-type": "text/event-stream"}) + respx.post(f"{BASE}/api/v1/responses").respond(content=sse, headers={"content-type": content_type}) stream: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) chunks: Final = tuple([chunk async for chunk in stream]) assert chunks[-1].type == terminal_type @@ -392,6 +403,72 @@ def test_missing_bigmodel_key_does_not_use_other_provider_credentials( call() +def _audio_file() -> io.BytesIO: + audio: Final = io.BytesIO(b"RIFF0000WAVEfmt ") + audio.name = "audio.wav" + return audio + + +@pytest.mark.parametrize( + "call", + ( + partial(litellm.speech, model=f"bigmodel/{MODEL}", input="Hello", voice="tongtong"), + partial(litellm.image_generation, model=f"bigmodel/{MODEL}", prompt="Hello"), + lambda: litellm.transcription(model=f"bigmodel/{MODEL}", file=_audio_file()), + ), + ids=("speech", "image_generation", "transcription"), +) +@respx.mock +def test_openai_routes_bigmodel_does_not_serve_send_no_request( + call: Callable[[], object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + upstream: Final = respx.route().mock(return_value=httpx.Response(200, json={})) + with contextlib.suppress(Exception): + call() + assert not upstream.called + + +@pytest.mark.parametrize( + ("code", "error_class", "status_code"), + ((401, litellm.AuthenticationError, 401), (1113, litellm.InternalServerError, 500)), +) +@respx.mock +def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( + code: int, error_class: type[Exception], status_code: int +) -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json={"code": code, "msg": "Token expired", "success": False}) + with pytest.raises(error_class, match="Token expired") as raised: + litellm.responses(model=f"bigmodel/{MODEL}", input="Hello") + assert getattr(raised.value, "status_code", None) == status_code + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_stream_answered_with_a_failure_envelope_raises_its_status() -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json={"code": 401, "msg": "Token expired", "success": False}) + with pytest.raises(litellm.AuthenticationError, match="Token expired"): + await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + with pytest.raises(litellm.AuthenticationError, match="Token expired"): + litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_responses_stream_answered_with_one_json_response_replays_it_as_events(is_async: bool) -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + request: Final = {"model": f"bigmodel/{MODEL}", "input": "Hello", "stream": True} + chunks: Final = ( + tuple([chunk async for chunk in await litellm.aresponses(**request)]) + if is_async + else tuple(litellm.responses(**request)) + ) + assert chunks[-1].type == "response.completed" + assert chunks[-1].response.output_text == "Hello" + assert chunks[-1].response.usage.total_tokens == 30 + + @pytest.mark.parametrize("event_type", ("response.failed", "error")) @pytest.mark.asyncio @respx.mock @@ -633,9 +710,15 @@ async def test_messages_stream_preserves_native_events() -> None: }, {"type": "message_stop"}, ) + native_events: Final = tuple( + f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" for event in events + ) content: Final = ( "id: bigmodel-event\nretry: 1000\n" - + "".join(f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" for event in events) + + native_events[0] + + "event: ping\n\n" + + "".join(native_events[1:]) + + "data: [DONE]\n\n" ).encode() upstream: Final = _ByteChunks(tuple(content[index : index + 7] for index in range(0, len(content), 7))) respx.post(f"{BASE}/api/anthropic/v1/messages").mock( @@ -652,11 +735,8 @@ async def test_messages_stream_preserves_native_events() -> None: stream=True, ) output: Final = tuple([chunk async for chunk in stream]) - parsed: Final = tuple( - json.loads(line.removeprefix("data: ")) - for line in chain.from_iterable(chunk.decode().splitlines() for chunk in output) - if line.startswith("data: ") - ) + lines: Final = tuple(chain.from_iterable(chunk.decode().splitlines() for chunk in output)) + parsed: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines if line.startswith("data: {")) assert tuple(chunk["type"] for chunk in parsed) == tuple(event["type"] for event in events) assert tuple(event["delta"] for event in parsed if event["type"] == "content_block_delta") == ( {"type": "thinking_delta", "thinking": "先思考"}, @@ -668,6 +748,8 @@ async def test_messages_stream_preserves_native_events() -> None: "web_search_requests": 0, "web_fetch_requests": 0, } + assert "event: ping" in lines + assert "data: [DONE]" in lines assert b"id: bigmodel-event\n" in output[0] assert b"retry: 1000\n" in output[0] assert upstream.is_closed