From 960d891522875ac06c8ffcd10048a27f77fbd634 Mon Sep 17 00:00:00 2001 From: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:13:22 +0000 Subject: [PATCH 01/10] feat: add BigModel provider with native Chat, Responses, and Messages APIs Co-authored-by: Bytechoreographer --- .github/workflows/test-unit.yml | 1 + examples/bigmodel.md | 61 ++ litellm/__init__.py | 5 + litellm/constants.py | 1 + .../get_llm_provider_logic.py | 7 + litellm/llms/bigmodel/__init__.py | 0 litellm/llms/bigmodel/chat/__init__.py | 0 litellm/llms/bigmodel/chat/transformation.py | 55 ++ litellm/llms/bigmodel/common_utils.py | 38 + litellm/llms/bigmodel/messages/__init__.py | 0 .../llms/bigmodel/messages/transformation.py | 149 ++++ litellm/llms/bigmodel/responses/__init__.py | 0 .../llms/bigmodel/responses/transformation.py | 204 +++++ litellm/main.py | 4 + ...odel_prices_and_context_window_backup.json | 76 ++ .../provider_create_fields.json | 28 + litellm/types/utils.py | 1 + litellm/utils.py | 15 + model_prices_and_context_window.json | 76 ++ provider_endpoints_support.json | 17 + .../llms/bigmodel/test_bigmodel_provider.py | 725 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 20 + .../public/assets/logos/bigmodel.png | Bin 0 -> 11339 bytes .../CredentialModal.integration.test.tsx | 42 + .../components/provider_info_helpers.test.tsx | 12 + .../src/components/provider_info_helpers.tsx | 5 + 26 files changed, 1542 insertions(+) create mode 100644 examples/bigmodel.md create mode 100644 litellm/llms/bigmodel/__init__.py create mode 100644 litellm/llms/bigmodel/chat/__init__.py create mode 100644 litellm/llms/bigmodel/chat/transformation.py create mode 100644 litellm/llms/bigmodel/common_utils.py create mode 100644 litellm/llms/bigmodel/messages/__init__.py create mode 100644 litellm/llms/bigmodel/messages/transformation.py create mode 100644 litellm/llms/bigmodel/responses/__init__.py create mode 100644 litellm/llms/bigmodel/responses/transformation.py create mode 100644 tests/unit/llms/bigmodel/test_bigmodel_provider.py create mode 100644 ui/litellm-dashboard/public/assets/logos/bigmodel.png diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 76af64c8ccd..7a10f07899c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -235,6 +235,7 @@ jobs: tests/unit/llms/azure_ai tests/unit/llms/base_llm tests/unit/llms/baseten + tests/unit/llms/bigmodel tests/unit/llms/black_forest_labs tests/unit/llms/brave tests/unit/llms/bytez diff --git a/examples/bigmodel.md b/examples/bigmodel.md new file mode 100644 index 00000000000..1ae64bcbfc4 --- /dev/null +++ b/examples/bigmodel.md @@ -0,0 +1,61 @@ +# BigModel (Zhipu AI) + +Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL + +```yaml +model_list: + - model_name: glm + litellm_params: + model: bigmodel/glm-5.3 + api_key: os.environ/BIGMODEL_API_KEY + api_base: https://open.bigmodel.cn +``` + +One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API + +| Proxy endpoint | BigModel endpoint | +| --- | --- | +| `/v1/chat/completions` | `/api/paas/v4/chat/completions` | +| `/v1/responses` | `/api/v1/responses` | +| `/v1/messages` | `/api/anthropic/v1/messages` | + +`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix + +The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` + +```python +import litellm + +chat = litellm.completion( + model="bigmodel/glm-5.3", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "enabled"}, + reasoning_effort="low", +) + +response = litellm.responses( + model="bigmodel/glm-5.3", + input="Hello", + reasoning={"effort": "low"}, + store=True, +) + +async def messages(): + return await litellm.anthropic.messages.acreate( + model="bigmodel/glm-5.3", + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "adaptive"}, + output_config={"effort": "low"}, + ) +``` + +Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route + +GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters + +Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search + +The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices + +API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/litellm/__init__.py b/litellm/__init__.py index 9ec11dd1291..4221feb11ef 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -634,6 +634,7 @@ watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() +bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime deepseek_models: Set = set() tencent_models: Set = set() runwayml_models: Set = set() @@ -873,6 +874,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje xai_models.add(key) elif value.get("litellm_provider") == "zai": zai_models.add(key) + elif value.get("litellm_provider") == "bigmodel": + bigmodel_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -1097,6 +1100,7 @@ model_list = list( | text_completion_inception_models | xai_models | zai_models + | bigmodel_models | fal_ai_models | deepseek_models | modelscope_models @@ -1208,6 +1212,7 @@ def _build_models_by_provider() -> dict: "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, + "bigmodel": bigmodel_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "tencent": tencent_models, diff --git a/litellm/constants.py b/litellm/constants.py index 2901b7d8948..54ec501471f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1083,6 +1083,7 @@ openai_compatible_providers: Final[list[str]] = [ "xinference", "xai", "zai", + "bigmodel", "together_ai", "fireworks_ai", "empower", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 19c1d5c414c..2af44ad0eab 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -795,6 +795,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "bigmodel": + from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url + + ( + api_base, # rebind-ok: existing provider-resolution pipeline resolves the API base in place + dynamic_api_key, # rebind-ok: existing provider-resolution pipeline resolves the API key in place + ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) elif custom_llm_provider == "zai": ( api_base, diff --git a/litellm/llms/bigmodel/__init__.py b/litellm/llms/bigmodel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/__init__.py b/litellm/llms/bigmodel/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py new file mode 100644 index 00000000000..1f721d84dd6 --- /dev/null +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -0,0 +1,55 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.zai.chat.transformation import ZAIChatConfig + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url + +_SUPPORTED_PARAMS: Final = ( + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "temperature", + "top_p", + "stop", + "tools", + "tool_choice", + "response_format", + "thinking", + "reasoning_effort", +) + + +class BigModelChatConfig(ZAIChatConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + mapped: Final = { + **optional_params, + **{ + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in _SUPPORTED_PARAMS and key != "thinking" + }, + } + if "thinking" not in non_default_params: + return mapped + return { + **mapped, + "extra_body": {"thinking": non_default_params["thinking"]}, + } diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py new file mode 100644 index 00000000000..d17f478ed29 --- /dev/null +++ b/litellm/llms/bigmodel/common_utils.py @@ -0,0 +1,38 @@ +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit, urlunsplit + +from litellm.secret_managers.main import get_secret_str + +BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_API_PATHS: Final = ( + "/api/paas/v4/chat/completions", + "/api/anthropic/v1/messages", + "/api/anthropic/v1", + "/api/v1/responses", + "/api/paas/v4", + "/api/anthropic", + "/api/v1", + "/api", +) +_ENDPOINT_PATHS: Final = MappingProxyType( + { + "chat": "/api/paas/v4", + "responses": "/api/v1/responses", + "messages": "/api/anthropic/v1/messages", + } +) + + +def get_bigmodel_api_key(api_key: str | None) -> str | None: + return api_key or get_secret_str("BIGMODEL_API_KEY") + + +def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str: + resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE + parsed: Final = urlsplit(resolved_base) + path: Final = parsed.path.rstrip("/") + root: Final = next((path.removesuffix(suffix) for suffix in _API_PATHS if path.endswith(suffix)), path) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{root}{_ENDPOINT_PATHS[endpoint]}", parsed.query, parsed.fragment) + ) diff --git a/litellm/llms/bigmodel/messages/__init__.py b/litellm/llms/bigmodel/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py new file mode 100644 index 00000000000..2cac2d3b899 --- /dev/null +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -0,0 +1,149 @@ +import json +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Final + +import httpx +from openai._streaming import ServerSentEvent, SSEDecoder +from pydantic import JsonValue, TypeAdapter + +from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai_like.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + OpenAILikeAnthropicMessagesConfig, +) +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response + usage: Final = body.get("usage") + if not isinstance(usage, dict): + return dict(body) + server_tools: Final = usage.get("server_tool_use") + if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools: + return dict(body) + normalized: Final[JsonValue] = { + **body, + "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}}, + } + return normalized + + +class _BigModelMessagesStream(httpx.AsyncByteStream): + def __init__(self, response: httpx.Response) -> None: + self._response = response + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()): + yield _normalize_event(event) + + async def aclose(self) -> None: + await self._response.aclose() + + +def _normalize_event(event: ServerSentEvent) -> bytes: + body: Final = _JSON_OBJECT.validate_json(event.data) + message: Final = body.get("message") + normalized: Final[JsonValue] = ( + {**body, "message": _normalize_message(message)} + if body.get("type") == "message_start" and isinstance(message, dict) + else _normalize_message(body) + ) + metadata: Final = "".join( + f"{name}: {value}\n" + for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) + if value is not None + ) + return f"{metadata}data: {json.dumps(normalized)}\n\n".encode() + + +class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def is_native_messages_route(self) -> bool: + return True + + def should_strip_billing_metadata(self) -> bool: + return True + + def transform_anthropic_messages_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> AnthropicMessagesResponse: + normalized: Final = httpx.Response( + raw_response.status_code, + json=_normalize_message(_JSON_OBJECT.validate_json(raw_response.content)), + request=raw_response.request, + ) + return super().transform_anthropic_messages_response(model, normalized, logging_obj) + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict[str, object], # mutable-ok: inherited native stream API takes a request dict + litellm_logging_obj: Logging, + ) -> AsyncIterator[bytes]: + normalized: Final = httpx.Response( + httpx_response.status_code, + headers={ + key: value + for key, value in httpx_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + stream=_BigModelMessagesStream(httpx_response), + request=httpx_response.request, + ) + return super().get_async_streaming_response_iterator( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # native parent yields bytes but its iterator annotation omits the element type + model, normalized, request_body, litellm_logging_obj + ) + + @staticmethod + def _translate_reasoning_effort_to_anthropic( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _translate_adaptive_effort_for_non_adaptive_model( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + def validate_anthropic_messages_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(api_key) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")}, + "x-api-key": resolved_key, + "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, + "content-type": "application/json", + }, api_base + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return get_bigmodel_url(api_base, "messages") diff --git a/litellm/llms/bigmodel/responses/__init__.py b/litellm/llms/bigmodel/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py new file mode 100644 index 00000000000..a61f006371e --- /dev/null +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -0,0 +1,204 @@ +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import JsonValue, TypeAdapter +from typing_extensions import Never + +from litellm.exceptions import AuthenticationError, UnsupportedParamsError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SUPPORTED_PARAMS: Final = ( + "input", + "model", + "instructions", + "stream", + "temperature", + "top_p", + "max_output_tokens", + "tools", + "tool_choice", + "reasoning", + "text", + "prompt_cache_key", + "previous_response_id", + "store", + "extra_headers", + "extra_query", + "extra_body", + "timeout", +) + + +def _normalize_output_item(item: JsonValue) -> JsonValue: + if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): + return item + content: Final = item.get("content") + normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item + if not isinstance(content, dict): + return normalized + return {**normalized, "content": [content]} + + +def _normalize_response( + body: Mapping[str, JsonValue], +) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary + usage: Final = body.get("usage") + output: Final = body.get("output") + tools: Final = body.get("tools") + tool_choice: Final = body.get("tool_choice") + input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None + output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None + normalized_usage: Final = ( + { + "total_tokens": input_tokens + output_tokens, + **usage, + } + if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int) + else usage + ) + normalized: Final[JsonValue] = { + "parallel_tool_calls": False, + **body, + "tools": [] if tools is None else tools, + "tool_choice": "auto" if tool_choice is None else tool_choice, + **({"usage": normalized_usage} if "usage" in body else {}), + **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), + } + return normalized + + +def _normalize_stream_part(part: JsonValue) -> JsonValue: + if not isinstance(part, dict): + return part + if part.get("type") == "output_text": + return { + "annotations": [], + "logprobs": None, + **part, + } + if part.get("type") == "reasoning_text": + return {"reasoning": part.get("text", ""), **part} + return part + + +class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BIGMODEL + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return False + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS} + + def validate_environment( + self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict[str, str]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **headers, + "Authorization": f"Bearer {resolved_key}", + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + return get_bigmodel_url(api_base, "responses") + + def transform_response_api_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + normalized: Final = _normalize_response(_JSON_OBJECT.validate_json(raw_response.content)) + normalized_response: Final = httpx.Response( + status_code=raw_response.status_code, + headers={ + key: value + for key, value in raw_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + json=normalized, + request=raw_response.request, + ) + return super().transform_response_api_response(model, normalized_response, logging_obj) + + def transform_get_response_api_response( + self, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + return self.transform_response_api_response("", raw_response, logging_obj) + + def transform_streaming_response( + self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging + ) -> ResponsesAPIStreamingResponse: + chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk) + response: Final = chunk.get("response") + item: Final = chunk.get("item") + normalized: Final[JsonValue] = { + **chunk, + **({"response": _normalize_response(response)} if isinstance(response, dict) else {}), + **({"item": _normalize_output_item(item)} if "item" in chunk else {}), + } + event_model: Final = self.get_event_model_class(str(normalized.get("type"))) + match chunk.get("type"): + case "response.output_text.done": + return event_model.model_validate({"logprobs": [], **normalized}) + case "response.content_part.done": + return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))}) + case "error": + error: Final = chunk.get("error") + error_fields: Final = error if isinstance(error, dict) else chunk + normalized_error: Final[JsonValue] = { + **normalized, + "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"}, + } + return event_model.model_validate(normalized_error) + case _: + return event_model.model_validate(normalized) + + def transform_cancel_response_api_request( + self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str] + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400) + + def transform_compact_response_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: Mapping[str, object], + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400) diff --git a/litellm/main.py b/litellm/main.py index 23eac8ce0aa..2f4ed844965 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2666,6 +2666,10 @@ def _complete_custom_openai( # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there api_base = _resolve_openai_api_base(api_base) + if custom_llm_provider == "bigmodel" and not api_key: + raise litellm.AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) organization = ( organization or litellm.organization diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b820b058eb3..38c86d266fc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76005,5 +76005,81 @@ "high", "max" ] + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d96f2976508..885220ac959 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -4144,5 +4144,33 @@ } ], "default_model_placeholder": "cursor/claude-4-sonnet" + }, + { + "provider": "BIGMODEL", + "provider_display_name": "BigModel (Zhipu AI)", + "litellm_provider": "bigmodel", + "credential_fields": [ + { + "key": "api_key", + "label": "BigModel (Zhipu AI) API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://open.bigmodel.cn", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": "https://open.bigmodel.cn" + } + ], + "default_model_placeholder": "bigmodel/glm-5.3" } ] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7e8645fd0dd..374aad26bd0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4422,6 +4422,7 @@ class LlmProviders(str, Enum): JINA_AI = "jina_ai" XAI = "xai" ZAI = "zai" + BIGMODEL = "bigmodel" CUSTOM_OPENAI = "custom_openai" TEXT_COMPLETION_OPENAI = "text-completion-openai" COHERE = "cohere" diff --git a/litellm/utils.py b/litellm/utils.py index 781aa5f4fce..35ab33750e8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9133,6 +9133,7 @@ class ProviderConfigManager: LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), + LlmProviders.BIGMODEL: (ProviderConfigManager._get_bigmodel_chat_config, False), LlmProviders.OPENCODE_GO: (ProviderConfigManager.get_opencode_go_config, True), LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False), @@ -9325,6 +9326,12 @@ class ProviderConfigManager: return get_bedrock_chat_config(model=model) + @staticmethod + def _get_bigmodel_chat_config() -> BaseConfig: + from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig + + return BigModelChatConfig() + @staticmethod def get_opencode_go_config(model: str) -> BaseConfig: from litellm.llms.opencode.chat.transformation import go_chat_config @@ -9645,6 +9652,10 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig + + return BigModelMessagesConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.messages.transformation import OpenCodeGoMessagesConfig @@ -9876,6 +9887,10 @@ class ProviderConfigManager: if provider is None: return None + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig + + return BigModelResponsesAPIConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.responses.transformation import OpenCodeGoResponsesConfig diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b820b058eb3..38c86d266fc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76005,5 +76005,81 @@ "high", "max" ] + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e62b74aa87d..b02568fcda9 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3289,6 +3289,23 @@ "rerank": false, "a2a": false } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } } }, "endpoints": { diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py new file mode 100644 index 00000000000..f53673830b2 --- /dev/null +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -0,0 +1,725 @@ +import gzip +import json +from collections.abc import AsyncIterator, Iterator, Mapping +from functools import partial +from itertools import chain +from typing import Final, Literal + +import httpx +import pytest +import respx +from openai.types.responses import Response as OpenAIResponse +from pydantic import JsonValue + +import litellm +from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig +from litellm.llms.bigmodel.common_utils import get_bigmodel_url +from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig +from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +MODEL: Final = "glm-5.3" +BASE: Final = "https://open.bigmodel.cn" +KEY: Final = "test-bigmodel-key" +BASE_PATHS: Final = ( + "", + "/", + "/api", + "/api/paas/v4", + "/api/paas/v4/", + "/api/paas/v4/chat/completions", + "/api/v1", + "/api/v1/responses", + "/api/anthropic", + "/api/anthropic/v1", + "/api/anthropic/v1/messages", +) + + +class _ByteChunks(httpx.AsyncByteStream): + def __init__(self, chunks: tuple[bytes, ...]) -> None: + self._chunks = chunks + self.is_closed = False + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._chunks: + yield chunk + + async def aclose(self) -> None: + self.is_closed = True + + +@pytest.fixture(autouse=True) +def bigmodel_environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("BIGMODEL_API_KEY", KEY) + monkeypatch.delenv("BIGMODEL_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + yield + + +def response_payload() -> Mapping[str, JsonValue]: + return { + "id": "resp_bigmodel_test", + "created_at": 1, + "object": "response", + "model": MODEL, + "status": "completed", + "parallel_tool_calls": False, + "tools": None, + "tool_choice": None, + "output": [ + {"id": "rs_test", "type": "reasoning", "content": {"type": "reasoning_text", "text": "Thinking"}}, + { + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": {"type": "output_text", "text": "Hello", "annotations": []}, + }, + ], + "usage": { + "input_tokens": 20, + "output_tokens": 10, + "input_tokens_details": {"cached_tokens": 5}, + "output_tokens_details": {"reasoning_tokens": 3}, + }, + } + + +def test_provider_resolution_and_all_native_endpoints() -> None: + model, provider, key, base = litellm.get_llm_provider(f"bigmodel/{MODEL}") + assert (model, provider, key, base) == (MODEL, "bigmodel", KEY, f"{BASE}/api/paas/v4") + assert isinstance(ProviderConfigManager.get_provider_chat_config(MODEL, LlmProviders.BIGMODEL), BigModelChatConfig) + assert isinstance( + ProviderConfigManager.get_provider_responses_api_config("bigmodel", MODEL), BigModelResponsesAPIConfig + ) + assert isinstance( + ProviderConfigManager.get_provider_anthropic_messages_config(MODEL, LlmProviders.BIGMODEL), + BigModelMessagesConfig, + ) + assert "bigmodel" in litellm.openai_compatible_providers + assert "bigmodel" in litellm.provider_list + assert litellm.get_llm_provider(f"zai/{MODEL}")[3] == "https://api.z.ai/api/paas/v4" + + +@pytest.mark.parametrize("path", BASE_PATHS) +def test_one_base_selects_each_api_without_duplicate_segments(path: str) -> None: + assert get_bigmodel_url(f"{BASE}{path}", "chat") == f"{BASE}/api/paas/v4" + assert get_bigmodel_url(f"{BASE}{path}", "responses") == f"{BASE}/api/v1/responses" + assert get_bigmodel_url(f"{BASE}{path}", "messages") == f"{BASE}/api/anthropic/v1/messages" + + +def test_custom_base_and_environment_precedence(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BIGMODEL_API_BASE", "https://env.example/prefix/api/v1") + assert get_bigmodel_url(None, "messages") == "https://env.example/prefix/api/anthropic/v1/messages" + assert ( + get_bigmodel_url("https://override.example/prefix/api/anthropic/", "chat") + == "https://override.example/prefix/api/paas/v4" + ) + assert ( + BigModelResponsesAPIConfig().validate_environment({}, MODEL, GenericLiteLLMParams(api_key="explicit"))[ + "Authorization" + ] + == "Bearer explicit" + ) + assert BigModelResponsesAPIConfig().validate_environment({}, MODEL, None)["Authorization"] == f"Bearer {KEY}" + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_chat_uses_native_endpoint_and_maps_token_limit(is_async: bool) -> None: + route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "object": "chat.completion", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + } + ) + params: Final = { + "model": f"bigmodel/{MODEL}", + "messages": [{"role": "user", "content": "Hello"}], + "api_base": f"{BASE}/api/anthropic", + "max_completion_tokens": 10, + "thinking": {"type": "enabled"}, + "reasoning_effort": "low", + "extra_body": {"clear_thinking": False}, + } + response: Final = await litellm.acompletion(**params) if is_async else litellm.completion(**params) + assert response.choices[0].message.content == "Hello" + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert body["model"] == MODEL + assert body["max_tokens"] == 10 + assert "max_completion_tokens" not in body + assert body["thinking"] == {"type": "enabled"} + assert body["reasoning_effort"] == "low" + assert body["clear_thinking"] is False + assert request.headers["authorization"] == f"Bearer {KEY}" + assert route.call_count == 1 + if is_async: + health: Final = await litellm.ahealth_check(params, mode="chat") + assert "error" not in health + assert json.loads(route.calls.last.request.content)["thinking"] == {"type": "enabled"} + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_responses_native_body_text_usage_and_continuation(is_async: bool) -> None: + route: Final = respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + params: Final = { + "model": f"bigmodel/{MODEL}", + "input": "Hello", + "api_base": BASE, + "max_output_tokens": 1, + "reasoning": {"effort": "max"}, + } + response: Final = await litellm.aresponses(**params) if is_async else litellm.responses(**params) + assert isinstance(response, ResponsesAPIResponse) + assert response.output_text == "Hello" + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.total_tokens == 30 + assert response.usage.input_tokens_details.cached_tokens == 5 + assert response.usage.output_tokens_details.reasoning_tokens == 3 + assert response.tools == [] + assert response.tool_choice == "auto" + OpenAIResponse.model_validate(response.model_dump()) + first_body: Final = json.loads(route.calls.last.request.content) + assert first_body["model"] == MODEL + assert first_body["max_output_tokens"] == 1 + assert first_body["reasoning"] == {"effort": "max"} + assert first_body.get("store") is not True + assert "messages" not in first_body + assert route.calls.last.request.headers["authorization"] == f"Bearer {KEY}" + followup: Final = {**params, "previous_response_id": response.id, "store": True} + if is_async: + await litellm.aresponses(**followup) + else: + litellm.responses(**followup) + assert json.loads(route.calls.last.request.content)["previous_response_id"] == "resp_bigmodel_test" + assert json.loads(route.calls.last.request.content)["store"] is True + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_decodes_compressed_response_once() -> None: + respx.post(f"{BASE}/api/v1/responses").respond( + content=gzip.compress(json.dumps(response_payload()).encode()), + headers={"content-encoding": "gzip", "content-type": "application/json"}, + ) + response: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello") + assert response.output_text == "Hello" + assert response.usage.total_tokens == 30 + + +@pytest.mark.parametrize( + "reasoning_params", + ( + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}, + {"thinking": {"type": "enabled", "budget_tokens": 1024}}, + {"reasoning_effort": "low"}, + ), +) +def test_messages_preserves_native_reasoning_params(reasoning_params: Mapping[str, JsonValue]) -> None: + payload: Final = BigModelMessagesConfig().transform_anthropic_messages_request( + model=MODEL, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 2048, **reasoning_params}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert {key: payload.get(key) for key in reasoning_params} == reasoning_params + + +@pytest.mark.asyncio +@respx.mock +@pytest.mark.parametrize("web_fetch_requests", (None, 2)) +async def test_messages_forward_native_tools_thinking_and_usage(web_fetch_requests: int | None) -> None: + route: Final = respx.post(f"{BASE}/api/anthropic/v1/messages").respond( + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 20, + "output_tokens": 10, + "server_tool_use": { + "web_search_requests": 1, + **({"web_fetch_requests": web_fetch_requests} if web_fetch_requests is not None else {}), + }, + }, + } + ) + tools: Final = [{"name": "weather", "input_schema": {"type": "object", "properties": {}}}] + params: Final = { + "model": f"bigmodel/{MODEL}", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hello"}], + "tools": tools, + "thinking": {"type": "enabled", "budget_tokens": 50}, + "api_base": f"{BASE}/api/paas/v4", + "provider_specific_header": { + "custom_llm_provider": "bigmodel", + "native_anthropic_messages_only": True, + "extra_headers": {"x-native-message": "preserved"}, + }, + } + response: Final = await litellm.anthropic.messages.acreate(**params) + assert response["content"][0]["text"] == "Hello" + assert response["usage"]["server_tool_use"] == { + "web_search_requests": 1, + "web_fetch_requests": web_fetch_requests or 0, + } + body: Final = json.loads(route.calls.last.request.content) + assert body["model"] == MODEL + assert body["tools"] == tools + assert body["thinking"]["type"] == "enabled" + assert "input" not in body + assert route.calls.last.request.headers["x-api-key"] == KEY + assert route.calls.last.request.headers["x-native-message"] == "preserved" + + +@pytest.mark.parametrize( + "terminal_type,status", + (("response.completed", "completed"), ("response.incomplete", "incomplete")), +) +@pytest.mark.parametrize("include_logprobs", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_responses_sse_without_done_preserves_terminal_usage( + terminal_type: str, status: str, include_logprobs: bool +) -> None: + logprobs: Final = [{"token": "Hello", "bytes": [72, 101, 108, 108, 111], "logprob": -0.1, "top_logprobs": []}] + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": { + **{key: value for key, value in response_payload().items() if key != "parallel_tool_calls"}, + "status": "in_progress", + "output": [], + "usage": None, + }, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "delta": "Hello", + }, + { + "type": "response.content_part.done", + "sequence_number": 2, + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "Hello"}, + }, + { + "type": "response.content_part.done", + "sequence_number": 3, + "item_id": "rs_test", + "output_index": 1, + "content_index": 0, + "part": {"type": "reasoning_text", "text": "Thinking"}, + }, + { + "type": "response.output_text.done", + "sequence_number": 4, + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "text": "Hello", + **({"logprobs": logprobs} if include_logprobs else {}), + }, + {"type": terminal_type, "sequence_number": 5, "response": {**response_payload(), "status": status}}, + ) + sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + respx.post(f"{BASE}/api/v1/responses").respond(content=sse, headers={"content-type": "text/event-stream"}) + stream: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + chunks: Final = tuple([chunk async for chunk in stream]) + assert chunks[-1].type == terminal_type + assert chunks[-1].response.status == status + assert chunks[-1].response.usage.total_tokens == 30 + assert chunks[-1].response.output_text == "Hello" + OpenAIResponse.model_validate(chunks[-1].response.model_dump()) + assert chunks[0].response.parallel_tool_calls is False + assert chunks[-2].logprobs == (logprobs if include_logprobs else []) + assert tuple(chunk.delta for chunk in chunks if chunk.type == "response.output_text.delta") == ("Hello",) + assert tuple(chunk.part.text for chunk in chunks if chunk.type == "response.content_part.done") == ( + "Hello", + "Thinking", + ) + + +@respx.mock +def test_get_response_uses_bigmodel_url_and_normalizes_usage() -> None: + route: Final = respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + response: Final = litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", store=True) + respx.get(f"{BASE}/api/v1/responses/resp_bigmodel_test").respond(json=response_payload()) + retrieved: Final = litellm.get_responses(response_id=response.id) + assert retrieved.output_text == "Hello" + assert retrieved.usage.total_tokens == 30 + assert route.call_count == 1 + + +@pytest.mark.parametrize("provider_api", ("chat", "responses", "messages")) +def test_missing_bigmodel_key_does_not_use_other_provider_credentials( + provider_api: Literal["chat", "responses", "messages"], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("BIGMODEL_API_KEY") + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "unrelated-anthropic-key") + call: Final = { + "chat": partial(litellm.completion, model=f"bigmodel/{MODEL}", messages=[{"role": "user", "content": "Hello"}]), + "responses": partial(BigModelResponsesAPIConfig().validate_environment, {}, MODEL, None), + "messages": partial(BigModelMessagesConfig().validate_anthropic_messages_environment, {}, MODEL, (), {}, {}), + }[provider_api] + with pytest.raises(litellm.AuthenticationError, match="BIGMODEL_API_KEY"): + call() + + +@pytest.mark.parametrize("event_type", ("response.failed", "error")) +@pytest.mark.asyncio +@respx.mock +async def test_responses_failure_events_raise_instead_of_completing(event_type: str) -> None: + event: Final = ( + {"type": event_type, "sequence_number": 1, "code": "server_error", "message": "Provider failed"} + if event_type == "error" + else { + "type": event_type, + "sequence_number": 1, + "response": { + **response_payload(), + "status": "failed", + "error": {"code": "server_error", "message": "Provider failed"}, + }, + } + ) + respx.post(f"{BASE}/api/v1/responses").respond( + content=f"event: {event_type}\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + stream: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + with pytest.raises(litellm.exceptions.MidStreamFallbackError, match="Provider failed"): + await anext(stream) + + +@respx.mock +def test_responses_preserve_tools_multimodal_input_and_existing_usage_total() -> None: + payload: Final = response_payload() + route: Final = respx.post(f"{BASE}/api/v1/responses").respond( + json={**payload, "usage": {**payload["usage"], "total_tokens": 31}} + ) + input_items: Final = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this"}, + {"type": "input_image", "image_url": "https://example.com/image.png"}, + ], + } + ] + tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object", "properties": {}}}] + response: Final = litellm.responses( + model=f"bigmodel/{MODEL}", + input=input_items, + tools=tools, + tool_choice="auto", + text={"format": {"type": "json_object"}}, + ) + assert response.usage.total_tokens == 31 + assert response.output[0].content[0].text == "Thinking" + body: Final = json.loads(route.calls.last.request.content) + assert body["input"] == input_items + assert body["tools"] == tools + assert body["text"] == {"format": {"type": "json_object"}} + + +@respx.mock +def test_responses_preserve_explicit_tool_configuration() -> None: + tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object"}, "strict": False}] + tool_choice: Final = {"type": "function", "name": "weather"} + reasoning: Final = { + "id": "rs_test", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Planning"}], + "content": [{"type": "reasoning_text", "text": "Thinking"}], + } + tool_call: Final = { + "type": "function_call", + "id": "fc_test", + "call_id": "call_test", + "name": "weather", + "arguments": "{}", + "status": "completed", + } + respx.post(f"{BASE}/api/v1/responses").respond( + json={ + **response_payload(), + "output": [reasoning, tool_call], + "tools": tools, + "tool_choice": tool_choice, + "parallel_tool_calls": True, + } + ) + response: Final = litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", tools=tools, tool_choice=tool_choice) + validated: Final = OpenAIResponse.model_validate(response.model_dump()) + assert validated.tools[0].name == "weather" + assert validated.tool_choice.name == "weather" + assert validated.parallel_tool_calls is True + assert validated.output[0].summary[0].text == reasoning["summary"][0]["text"] + assert validated.output[0].content[0].text == reasoning["content"][0]["text"] + assert validated.output[1].model_dump(exclude_none=True) == tool_call + + +@pytest.mark.asyncio +@respx.mock +async def test_router_routes_one_model_configuration_to_three_native_endpoints() -> None: + chat: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "created": 1, + "model": MODEL, + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + } + ) + responses: Final = respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + messages: Final = respx.post(f"{BASE}/api/anthropic/v1/messages").respond( + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 10}, + } + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "glm-native", + "litellm_params": {"model": f"bigmodel/{MODEL}", "api_key": KEY, "api_base": BASE}, + } + ], + num_retries=0, + ) + chat_result: Final = await router.acompletion(model="glm-native", messages=[{"role": "user", "content": "Hello"}]) + responses_result: Final = await router.aresponses(model="glm-native", input="Hello") + messages_result: Final = await router.aanthropic_messages( + model="glm-native", max_tokens=10, messages=[{"role": "user", "content": "Hello"}] + ) + assert ( + chat_result.choices[0].message.content + == responses_result.output_text + == messages_result["content"][0]["text"] + == "Hello" + ) + assert tuple(route.call_count for route in (chat, responses, messages)) == (1, 1, 1) + assert all(json.loads(route.calls.last.request.content)["model"] == MODEL for route in (chat, responses, messages)) + assert ( + chat.calls.last.request.headers["authorization"] + == responses.calls.last.request.headers["authorization"] + == f"Bearer {KEY}" + ) + assert messages.calls.last.request.headers["x-api-key"] == KEY + + +@pytest.mark.asyncio +@respx.mock +async def test_chat_stream_preserves_deltas_and_final_usage() -> None: + chunks: Final = ( + { + "id": "chat_test", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": {"reasoning_content": "Thinking"}, "finish_reason": None}], + }, + { + "id": "chat_test", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": None}], + }, + { + "id": "chat_test", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + ) + respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + content="".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n", + headers={"content-type": "text/event-stream"}, + ) + stream: Final = await litellm.acompletion( + model=f"bigmodel/{MODEL}", + messages=[{"role": "user", "content": "Hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + output: Final = tuple([chunk async for chunk in stream]) + assert ( + "".join(choice.delta.content or "" for choice in chain.from_iterable(chunk.choices for chunk in output)) + == "Hello" + ) + assert ( + "".join( + getattr(choice.delta, "reasoning_content", "") or "" + for choice in chain.from_iterable(chunk.choices for chunk in output) + ) + == "Thinking" + ) + assert tuple(chunk.usage.total_tokens for chunk in output if hasattr(chunk, "usage") and chunk.usage) == (30,) + + +@pytest.mark.asyncio +@respx.mock +async def test_messages_stream_preserves_native_events() -> None: + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 20, + "output_tokens": 0, + "server_tool_use": {"web_search_requests": 0}, + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "先思考"}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "test-signature"}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "text_delta", "text": "你好 🌏"}}, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 10}, + }, + {"type": "message_stop"}, + ) + content: Final = ( + "id: bigmodel-event\nretry: 1000\n" + + "".join(f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" for event in events) + ).encode() + upstream: Final = _ByteChunks(tuple(content[index : index + 7] for index in range(0, len(content), 7))) + respx.post(f"{BASE}/api/anthropic/v1/messages").mock( + return_value=httpx.Response( + 200, + stream=upstream, + headers={"content-type": "text/event-stream"}, + ) + ) + stream: Final = await litellm.anthropic.messages.acreate( + model=f"bigmodel/{MODEL}", + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + stream=True, + ) + output: Final = tuple([chunk async for chunk in stream]) + parsed: Final = tuple( + json.loads(line.removeprefix("data: ")) + for line in chain.from_iterable(chunk.decode().splitlines() for chunk in output) + if line.startswith("data: ") + ) + assert tuple(chunk["type"] for chunk in parsed) == tuple(event["type"] for event in events) + assert tuple(event["delta"] for event in parsed if event["type"] == "content_block_delta") == ( + {"type": "thinking_delta", "thinking": "先思考"}, + {"type": "signature_delta", "signature": "test-signature"}, + {"type": "text_delta", "text": "你好 🌏"}, + ) + assert parsed[-2]["usage"]["output_tokens"] == 10 + assert parsed[0]["message"]["usage"]["server_tool_use"] == { + "web_search_requests": 0, + "web_fetch_requests": 0, + } + assert b"id: bigmodel-event\n" in output[0] + assert b"retry: 1000\n" in output[0] + assert upstream.is_closed + + +@respx.mock +def test_stored_response_lifecycle_reuses_base_and_unwraps_id() -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + response: Final = litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", store=True) + listing: Final = respx.get(f"{BASE}/api/v1/responses/resp_bigmodel_test/input_items").respond( + json={"object": "list", "data": [], "has_more": False} + ) + deletion: Final = respx.delete(f"{BASE}/api/v1/responses/resp_bigmodel_test").respond( + json={"id": "resp_bigmodel_test", "object": "response.deleted", "deleted": True} + ) + assert litellm.list_input_items(response_id=response.id)["data"] == [] + assert litellm.delete_responses(response_id=response.id).deleted is True + assert listing.call_count == deletion.call_count == 1 + assert deletion.calls.last.request.headers["authorization"] == f"Bearer {KEY}" + + +@pytest.mark.parametrize("operation", ("cancel", "compact")) +@respx.mock +def test_unsupported_responses_operations_fail_without_an_upstream_request(operation: str) -> None: + call: Final = ( + partial(litellm.cancel_responses, response_id="resp_test", custom_llm_provider="bigmodel") + if operation == "cancel" + else partial(litellm.compact_responses, model=f"bigmodel/{MODEL}", input="Hello") + ) + with pytest.raises(litellm.UnsupportedParamsError, match="BigModel"): + call() + assert respx.calls.call_count == 0 + + +@pytest.mark.parametrize("model", ("glm-5.3", "glm-5.3-flash", "glm-5.3-flashx")) +def test_bigmodel_model_metadata_and_cached_token_costs(model: str) -> None: + entry: Final = litellm.model_cost[f"bigmodel/{model}"] + info: Final = litellm.get_model_info(model=model, custom_llm_provider="bigmodel") + assert info["litellm_provider"] == "bigmodel" + assert info["max_input_tokens"] == entry["max_input_tokens"] + assert info["max_output_tokens"] == entry["max_output_tokens"] + assert info["supports_reasoning"] + assert info["supports_function_calling"] + assert bool(info.get("supports_vision")) == bool(entry.get("supports_vision")) + prompt_cost, output_cost = litellm.cost_per_token( + model=model, + custom_llm_provider="bigmodel", + prompt_tokens=100, + completion_tokens=20, + cache_read_input_tokens=40, + ) + assert prompt_cost == pytest.approx(60 * entry["input_cost_per_token"] + 40 * entry["cache_read_input_token_cost"]) + assert output_cost == pytest.approx(20 * entry["output_cost_per_token"]) + assert prompt_cost > 0 + assert output_cost > 0 diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index cc61b6f2fcf..f99c9da5d05 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -25,6 +25,26 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreate from litellm.types.utils import LlmProviders +def test_bigmodel_fields_require_private_key_and_share_default_base_with_adapter() -> None: + from litellm.llms.bigmodel.common_utils import BIGMODEL_API_BASE + + app: Final = FastAPI() + app.include_router(router) + response: Final = TestClient(app).get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = TypeAdapter(list[ProviderCreateInfo]).validate_python(response.json()) + provider: Final = next(item for item in providers if item.litellm_provider == "bigmodel") + fields: Final = {field.key: field for field in provider.credential_fields} + assert provider.provider_display_name == "BigModel (Zhipu AI)" + assert fields["api_key"].required is True + assert fields["api_key"].field_type == "password" + assert fields["api_base"].required is False + assert fields["api_base"].default_value == BIGMODEL_API_BASE + assert fields["api_base"].placeholder == BIGMODEL_API_BASE + assert provider.default_model_placeholder is not None + assert provider.default_model_placeholder.startswith("bigmodel/") + + def test_fuse_presets_route_serves_the_shared_catalog_without_authentication() -> None: app: Final = FastAPI() app.include_router(router) diff --git a/ui/litellm-dashboard/public/assets/logos/bigmodel.png b/ui/litellm-dashboard/public/assets/logos/bigmodel.png new file mode 100644 index 0000000000000000000000000000000000000000..f22030f85b8fe85456e8abd027952c44edf16227 GIT binary patch literal 11339 zcmWk!2|QHa7k{&07|S60zEjyLgc)nL5VA#xA}Pwg#MqaFC_7`xmO_by7?dAT_9cn2 zZ`rpoX8F(m``)|n^Ld~1?mhS1dzO3dO|~?@&cwjW0001!v5~$N^^N`i1A|b@d*DJ< z0Du6NW;O=b{mir+hPZ(o3WahfiK1lt+vCON)jP{d*24+!Ns5ktNMwrlt3B<2H5I4X z(=#%aj7XyRX6-57{=2?SbbaxUa86OUJ%4mc^2%AcQ?$FiPZG14yYhG$w?^DQCOL;~ z%Q?>J-dl7C+;=Fjnd*jyQt0U z)iZ5R;?`W(AYS+0ica>+s5eJm!TZSjD*>^TjsX^gdNYwGOsvRqkzLZ$QxKF}*(le#r`=k2z6 z zT^BzO-~+Nx65`Li(s!xf-(ydwapyO}HWS{SIVSGt6tC9(I@2jyjjAGeXB^&7J8+2K z`Z9iYr(& z?^hn!#cn_Ov~&B-Zd}}1Ml;c?@-V$;KR4%WWa&Jt;H+W%lsds|%6F_D{<~53&*{xx zNaU$c$l;UlV~@&%>-RSvRvp_t+BWmryzy+yF=5xBbVJvDP0MZ7CT3f~aq+hAPGmIR zFm%)9$*#Nqo@dZOKn*^kmf(`SFJp&`iz7V9IDV9M5|oQ~e}15eS-qcj2pP%fqN>E7 z04sBA00KG(!{B5ZG9w;FX5A;9u<>nhABykNm~E2CEB}OKIdw^*%+i|1N5_it#I;qj zmfarD)%}yP^~0rq?$R2^@5T?7kHAZKe_U$y(e3r|mH4@?-JQjwp@=!$IeyV%b+Myu ze`$HpduRJ-b$ev-6P*8jy^7M+H((vnhe9u0@H8BO zKLr`H@faXz@)#9ptyLffp%0R{SFH_Abs>sE%oEniO;5WA4^(sAF4F&cEup@0{;ug% z+Y@t*dl4aPHSf|sA558a3|Cv5`&-)M&ho`cZtCU6z>!0rhmyb-r#YRu1q|x}^V2Ws zX=ajH(E-rC{g)K$@nB)*62G~as1(Rk$va%#SS&zRR2f$to60!@^ut+%zA%2{1WEquhEFDNX;NWZ=cK;WG}Zf} zv8hQx@iE3h{J#fqM=2>vJlCCE)rZ}2^aDBdPG@uB%6Iq@VZCVAO5aMcv6{z70pmKo z+fp7ZSuC4Xv^wuXgU=35=OFC-Sg1RnmX@BLa-RA52VG`&mdzR>hj7&9_|Jl?zRV07 z&G&!itL}E3*P&570Y4)&K4qauHe@ez^2`_iwO_|hAD6M26yh3^KSp*CekH7R`<~U_ z%Z<#*o&lG(pzYaYb}cGC3#s|xYU>dTTEA#a`LswuZqZJa@5(_^k;(q)6?`+0HmaB)S93Kmt1eNv~4x}uP*`<1a>XSrX8yg!ZviQT4 z-}S1ys{i5c*0}Xw6=BzYxhwm7Du+A|nWK=H_e9zAS8CF882uAtQ60o&dt1bciGD-8 ziqi1J#MP_~BGZ7)m-K1%w!?!pt9@lL1zjg6Cr&8|C@n2*8IvejwF_l8mAW%k+!6V4 zUHttaO}c{(c`#4@B?`=y^ZhxyFnYmnWuZRX66zhJ*jgeV-U@%DE`{MEf62hh8?>tk zELLpjBSA1Y9Fp+nO%GkiBFu+X$HLA|sM*KDxZSC)Tr>*DknX_lieI{Kbu3;6L+R=s z2*W&pIEjaomtD*+bxoGFRlEfGFUTnL#gBNhbN#@yhRTiYE-$OhZQKJ;bS9jMtzVdU zRn|F$#l^*wpwKQ5YidIw#!#^T>=%8%-m*JEU*C+5v{WQK9Q{I0PC|mdgtCT->WGI% z8yml-Q-fYg0iYkHkWDX-N)PKMupEMd?2^z&#n-0Bl+!kXH>X)at7hpbPeF75&NE2| zPRFFna*-bF)}RYD730%#R{m{0JT%FqK%o8?b0wD8qUKdFRy8C-s24%mT*yY_0Ab;- zyD3SGJ-xCg{Xk4qw5)7(@X1pv4d;zMm=~=Mmjr-7 zipKCIHqe^cGf3usGy#(SGZU%L?#~T_dlhAn)QBEOIvrLro4f08E-)Dx-3O3l{8IWF zfR(*~;DmIuq4ofFLw^#Cxl8a8utoeUTXLGQ$iz@aZLa8*`HxE~8bLO9X}A5K7+<6T z0{|3I3*ITj6x{ZbMqHMJ>I~$BUzUJX{*6yKQlnq?==XK=-cp!h$i)0OO@-@_Oe(BG z&ki`GX#EV5nOj;~#Je;Zfx#H60(~huJoG<=Gs&{vfI03gdiv?s4;Fn%S|KOmP}U$^2l+-`d#VVe4KlClBz)JpEO8>*l!=rvwOf1FV4p zEIa2NkV(Uq!hZBMvmGA&A4*E-*{SA+@l)6=beN#{jNcG`kb^jryGezKXT`@4ANHS$ z=?D{E|ERuoe)3vU*2M8bZU!QO(}-U!{NyuVBm>gGxc_bp&Y0EqG6&yi`TMlKw1HGJ zCr-|0W?NgfGdbb;-wqp$2YfWp;V6fifICfZ-vsQJiPxj*^Le58n|?U{b>#UU->Z+M zFEF6|p3u|R9ErnX=-*Xs4!r$y;s<38AMDGfSx#@x|1jR;9%?!KK|;U7)W#;_)#&dp zHOXKV20>w96VrJw()EU}J@KaynToj7(g74~MB4Mk3iX#P&$A=^+a3QEte+NLbjUR~!6LdNteJ)=jLWg;; z5T)mT|2|6;%FllT6GeGDf=66be~-M>5FTLxI_L1mQ(IH=(uGS)OXVhIPENXux%7gU zXqC8TP{N7s_aUaHI%E?K;_D2egfxYYO&=*HOyDb8TGAUE^bL_U(GXhzdDQdMm$|y1~H`#KTiK+&q=0+`N-VPo^~fIudnxCPGx z79Xh|7d9b_W$QS}WctDA;oS9X@;ahqg}q4B_)Z$3S_`1HnUd> z7WM~X3$b+n?k9xlu5qo{#Sp(sNJ#9bz`5#NI{lpgLZVAX99Z+6mW22n?CtDYsFD^w zm;lM)wbL#fuTA}i;ETO!ugF(EYDsm;3XtdLgIC! z3HuZ-`g|LJw*VomcIByqv$ONx&A$FD0TA|v3hrtoQQq>Bn{9wCSP!azh;xu{N8P|( zg*zhN2726Y4ZC6}sdD}VUlYMiWLuv}fqQ8LqJBdBuq+vxAPA3 z1OL1;k;vd_Z?~XpivrkbnT5;>3mqrV_py_pCLEn0iZx%ecS6fkC&a+kAl@~)pg`z{9R#m~ zr+Hy=gh1fz8DW!;i!~lj!Xb1@?5>UF9H=R5E;G`uz1pR8ulz-URg$0)tK8L#rO+R} zgW6V@+b;u;ybSP4R{sIG|F)`*!=p|0CR5z>Ky>cPOOob=ZQ1z%yRtbw;i>E&Rb@TC z;q-&4;eW=C?Nr{pq&KTX!b&6o4z>R-&00Bm3e~Wjn$I?GSG$7L(9nw3%69X9{qG;YP(ADhvmS&^ifMbKCqmE> zVGOeIY=CDTu!1y#;`FqIL-G&^^a!c#Ht&OAZ0>!;I^Egh$B!c)`+!i0$FM72KSbBy zzZeRk)gUdx5x2&!7w4p+4EP42;2V^!xdJ`>VX5iiRIe#Lp=kl~B$2A$-535zgFi9v5HM$7>9N$3a`N!`T zyFI7z*g|(8jq`P7NLb*;ME#KEyQPWiSg;J*gzlCsAdZh^vxFO-UX@Ue4nhZ0{Dozr zi-D|)JqJVp{FZ9 zYC}K${DK;}KP~G6>ujsv+a!&=M>vgD(E7p>=3RtspFDW6IqsH5d=hO?#NBSIVd& zAnklp7L6Q^z7DurtO~v|1WM426_2B_77MEhgm_ixNV?8th4lXJd#X1? zwMl%ce5hVZ`r6DgD)%C~k{)DX6xp_4fax9@Vmk0tGIMK5Sf8Ld4&I(=+EO0U9DDC6 zU-%O(%niG+&$Z{#cJm&+xN6BZ5Y0yS$&*zBeJ-Fu_F(duD|g8$Rv?xL!8A%~0)ma{ z3x7+J5AOXW+|*k4g`;$*-{1bs4Vq6=F?-M+nU8uFcKnZav-7|8nI?97Z1a^5U1|MB zD4`uETTGW1y)t@jW)Uk}jMBSW21J$!9UxOTPt!{2#n&n1^E$8ShPpJSUL_#L@b!)5 z*Z-}cgf*~wUcW>;8zc8-wbadq)7x_8ynrzZ>C_A4F#hr<51TM{W#DbNLJheYjG%jS z#Y-1^#G`*CuSfa>(dyZtgG=j_+4q%tb=|w!N#Gw;s=mR@c$3Dm78)|MzDBzwzl_WS zmBfUS=7aBjwGrNvil2?+OgiYlk~=j&$;+*@%&CWWxCZvQws22dhoUKslY2|60RGtU zi@}A{7`7OV3S^h>7kV{+g)Sk9Q-M9w#<)CwjZ@A^OvSJ`~~Ip*vlhMiw)$GBYBUl!P1cFS=URV4i|WL|1rw9?w$2a$SamSVs3B+K8sWN zcf-?=T4>NvD;n4LNj^g#_XY)EnO)Edw)0NM6YH}ft)sMmey%tpNf&XP7C>SWIF3|6RrEVv#i9j2)Tv9HFJ-1|&0E>CB?uDG;-~$XH(Ua!`hWhc7xXgTovy z@&XYMu(;AlK3iutf_>u-c+Rn_bQ3u#?;PU|ASxkN{g$v?!&kTzm1MyxNM;&?`78sU z($hBzM2-w?y$XUVWY0lGK0(arMY`QZV~OWmYID)oK?wKd3=ucIZd|>{iCE{E1~QW<#hfbO$n3~mJ6@ochWP%dcLzO)X)eFXuU$733<7&aP`=QhBoa2%rK2# zDHBS*wp<`GkK$K<8rO$xWvXw>%&@3#> z&1H~swclqCF@S>9TH1?q)&+`hyF_`M_)ICfdpB2BRDd8lxq7T9im9@c`RptM8pMba zFWGb#5f!r&a`?F{CE1H2G)zEjeIEiz@HdDs)KyW~PNTCkiS%%k3KJ{4WSIF_zrS|K zmg{6nB5kf2kVNR4sQs`DKX`E5O-q(CB(^W*3m%l`5F<87G^9`k&ztvb4~LqQc6ncT8no_ z&QzHJR3t@VFnOAr^*YFq(gcBG6^BQjyeVJuu-~!y-rL*VW)~P3*wn-^t!{i!LcFpr z6*znYr9j7b@gnCq2Pfx!D%Y+(Z>j5^7x%sb^PRi6=+v=FYEz`Pw8ZzfB6vfb)GO)&=%Z5 zT1c$V;65}_@2zB)GB3namx`T|&~?zy((B;$e%qiJ(97Q+I=brW>J-p2RNTA$c`_t) zdzSRQj%n+9`<2IgE+FF8&!|oc`HvaWFf{b6^??0V$;o(4EL|tb)?TMkc|Fl?i1|6; z31N%DU-a)Y&$iaq)|qr;;|~@e%#DqC8YiO2*|Ke{*t%PQ7gi<7uJY@FN$({do|Ua1 z19^0)1}6GGK{mm=pWHrmrM!I{=>sO7&&}CM<-N05@uMu%*V5 zfxh*$if%`|X%KWV3#&7X$;w*Vkz4=V@i6LCj>uv+g*&_55iSVac_fQfG4?>gpe`7V zz|Ap>&LWin=^`dS`Qab*(b@;-6PX{#u!gecX1OQ_^EcPHUDKh#+lfqV*Lyh=|Sc>q5^o9*kP{iP`^ZDn0w!7IY|6H{}q0a$kh1xGv)?}e68#%c={73y7$wAF}YHDl^5pi`^Q^CQACia*`^NS z6Nc`=5228t%a>!?-9)5bM@Q`1o19>|2w;U$A0ftJv)0tdGmdj!Hr$TS34dD$q_fTyguKjK_6@j zvEtZ>C=VxZ1Y+_lUQ{@O*hL2hg*Kti=TeJ5z?U$&9K{9jgC~16Vc}bClas-Uf*42~ z@U&W_TtrMmLnG{OD>@K_MoUKg7^0uKftZovH^Ry>CLj}c&()qd+@62UHW>Lyw6Z*E z!>UIGUn^2xDw40H!0xC6R0ReQ`>$lK59yrM32T4Xi0|V9#mRY4V1RIusEUe;2iAp& zsmel37lk&$s*2sXtuh+rM|^PZMsRi;aeq|wbnT?*8kHj`j1^T#H;L3lmyky*Xc|O&=WfZY>ZS|dwj=H**<@TualkGoP;DQ%euO-~h&$Xuql0@LE4l&7y zhK_)EVJh10cS(t#DEh;gr5)x29SZ6ElpaJc_afo%_laRKV2{Aly^%0h#iukJi$=3= z2BHTTR7uM>75OL;cPLs6Sk02;Z;(M>c50X{N>M?m6lxbeNto<<{?_3Q2ykibl7eKh zA-j(}53G0ovV*E2!7U~o4}U_YwHN^@p~#l~M6aX-t2jfg`e(-fpieu_9F95bO5Cp@S{m3$nPJBWC-V>nsNvAmBTv$Ml#WMyT`Xij|(A9y_bkH8GJ zXGc4;?4m!BV83HWYd{y!QYD;zV~W~C8SDQobcXCN8ob4T5#UU{;2kt1!|3D@*wPnM z2C~sgJ{$q9eG$)BNe*|9VH*x!RU^sVqoLuVCRWg*Mw@6=F%{wOk%MA@-&qeIb{wM~ zMb>OYv+YS4zM*^hnX~i8AVU>N=;$+={9N`eR5PYYHIN#SXPyjg<|yNcd50rw&GmzF{eTZj>A%i;$Et3na~(+8cd)1P!^eQg9z5RiXTFWQsS{EHH5g1+upa*c{Rr8B+sqlw{eb6DdnXoxh*b@Fma_@>Ob49iK z@Y`|&CfdT%0lFG8rZ3$P-Wn+8%TWT#MO9xSrZufD&ja%FGl6K9NI;9?kckn0|~= zvsjb=fi9P~-|OY8UjqZr;$LZ_^Q5>fEiGqfBRC?BcwC;L>5cr(nKf_*Aji`5mtX$uOb5pAsBSmcz3|xIKzq?y2OSKkuLWVC?oQ z#oJ$DC|_v9i1I5f1a9D~YxF0|bLJ$H^?2G6Wo*|~N77z+T=7h0Yb3>S1l@k0O;^O} z!l+T83HCQG(&WeF`#X2qf_EJ$cYNEPxXY$KIvP}gaEoI1uXc)ameBDkCGBF^NGgeQ zeM1$uiIF5DumBViQ3JYoAYuc8vOK27Ih2lE9pA6_uTD>bD?zeAFNF3~Fk1cSBynZW zHTB-h0Mg~jdVSI^4i#3}a53IcA2}k+P3_2ld}k8ur?g8gy*0EB*p&c2`|s9+!R zi{PeSsE?uACk?$njFK1REfiM1n>8zP&y=%7R+3#G2`B=f653oQ6@8MdC~!l(Oi0PA zC#+A{v4pw522w@*dybRuFT~hGV_R) z8;x5w?usehYku}P(!9BrGN~1w!ixHYu6IAym3}C5DR%9s+9j8!Po?s#V2$yS;R^}% zqutZ_`SX)M1vr7&)YSD^<2Wl^Pr9Qc2Hw(!?-7?a5C=P>>*`?@~a zVA#dhw)bdvhGI({nGh!0$Xl-^0cE}{&|r-G{QLmo$9`VI4_?Rjks^<~F$Eh##hS)K zKz>1#sa@$ewC4EG5Bmx(XAaGlX)C`E=st_XgPCN4NNg`t^Y;qrh`;TIfR;P_6AoWM zA)n^#`ol)BSYHrc9SxfOhB=xC0kTQSdC+>JwKY7g>G5;u&+fOno4TNN3sb$bn8@`~ z`))@NWxIeC|>EP$*zJ^}~b!!-jO{gMd z05_uVDDq7T62Ct7?OVAQdMo#RHB!F55r>t}zuaOZ%)oH+uPY}`|7<#%HW#Y|22>K$ zcn~N~5LijL`%e-Y>djzA@za{iQT$6Y%ytRIFx-UxxI11jGJ>x{G%X*YGKqBiSAFLqP8is8+&S?IHrz2Nh}oTPc&l*G!_l`Py8eOkIE0?nV;8$R90E zp*dp|K6S^ZDckE0LpM{SIT7q{-g!@W)=|KQK?7ld+ zs_W%f;K}a8cWez8z+fCA={_qgvD0e@5$EY^9)0oG&!1Yi;JX=LtRdBrBKfx%N=v?! zw3KjvtCEdbm?wYd^1%1sf~yhzZpT?Pom`8<-S++&zyp?vU-H$Kzo3kI2ZzC68NXbv zI)D@)B`GdLKWRvT@g3xL5P#eoE>*6pTvQw$1Pjb`Qr>?KV<&|~W$*|@oyB%u9!*^y zk@lu`&{vy9P9fo`g3f4UNg%`K!rJdFsV$eGp?V1EcNcjaUDn~mKraac5f+$CEm_Jf z5~31{VtW9%X-gW@@)o{NC?t{n&M(~#h0z3MofJz!fG@=ObPbNy$iQGQ>GC&el_8c$ zwtz{LrIS;k$x0OH!0pI^fI9rIO{D3kD_h*AH}%9sUg>ap16x{W58}$BgTMNjM%B*A zgn}Val-5M27@rc1Jqc{&<7GSIVshOjBb~5t&sV@^k{};B(WMCnCs3VN+h^}FNns-v zM3&KLG&WjBYQAr1C~@j_V=ZwED5>_J??^=tmAMQjXb!tLz7?HMv#EbkTo-w}Lx z8v4Da=3)=B$E1Dvd&3LDLjENqF}|gn>)o|qucxZX8T2v2j4Y}MZtKZm(CVSV{{COT z?gOC)2G2s_8nI!yIfn-aR>?#JcchN?vi8?w(*g2l4iI}|o=7%W535Ny7@$0B4yMxq zSVlM2vyMucJyu(kNe^mDFlyL9QG5I4Qu64tb6wqLU&4`LR&O zoKv0aF^(mY?P9_nV&HvrljSNv-F|T!wX%Mm3wd_!dEoFD>H4GyZOF>lLgMYp%ea3( z_CCygPJnOUgwV1?9#61-i(5ld<~=dn)juks)y1{ZW?+S=jww+=;D|k@ti7ykeEb&M znP9GVGpD3q>41BObs}4;0P*4lWzh(-dVa||B_)Z7 zFxUS2ve#&6@5vq4Xpx+}h4x+os{#IBqrkN!dh1HF1$rfAfS4v4Qp{`C>=ZG4Pby7UYO@*`}p^8 z`hzguk1ax@pHsdLy;@%XZ0Hy!l5>vFh`;hao>h!|{_9#QTGWyS8~wYZiwsyY02r8z zc%LWXG&}^iUeJx>ThCuG7!@eC!DHW{zoRDx3$O_DQARL8RN(_#k}wf52Mgt)BE@P% z_VvB7*-%mUA-ZW4y}}DDR1jpA8}mqIZ0a>+{oHsy?HMu}MT#NP>mwOEi|cl*zvQt5 zlkb{Nq$)`GpZGxcE^-JUR{k?y`Lmd#$fvyn&;1m~4>K~24}EbYR?5i@dV}zHy)`0r zd37-QpBwcM$m!Miw!Xfcw@4b2!1Ba|63^vyLt{j7)E)7@Z;Yae3$=l4f;FbLE}la7o^DW{>aFG z#yWwvc;dBT7NA0e3N!85L0vhSm_Pi&XMxMwQ9XY<*qDRR-wt4`#S~)OZpR<0$1G%f zJTFN1SwEWSuqa@vM%-R53!cgF5)=U|2!qy7k2HJdpZDZjw2jPzm@}qFzSoL&*EzR! zpw)u@O)CaTF-s!9wtx7$zKe!BA$8-H_^$X&Q@7Jf^%BzrVs5eg;ClQmdiBu5SMgaj zRymX-j+Nrv8QDW-N@Nz8LArs(k*_VrkxlUymmrRBnFrx_RZj&K?~(||=OUWZB(F+* zlk~I(rEczbYa={m_*|CF^!u*| { const actual = await vi.importActual("../networking"); + const { readFileSync } = await import("node:fs"); + const { resolve } = await import("node:path"); + const metadata = JSON.parse( + readFileSync( + resolve(__dirname, "../../../../../litellm/proxy/public_endpoints/provider_create_fields.json"), + "utf8", + ), + ) as { provider: string }[]; return { ...actual, getProviderCreateMetadata: vi.fn().mockResolvedValue([ + ...metadata.filter((provider) => provider.provider === "BIGMODEL"), { provider: "OpenAI", provider_display_name: Providers.OpenAI, @@ -157,6 +166,39 @@ const chooseProvider = async (user: ReturnType, provider await user.click(options.find((option) => option.textContent === provider) ?? options[0]); }; +describe("CredentialModal with BigModel metadata", () => { + it("requires a masked API key and shows the default base", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal(); + await chooseProvider(user, Providers.BIGMODEL); + expect(await screen.findByLabelText("API Base")).toHaveValue("https://open.bigmodel.cn"); + expect(screen.getByLabelText("BigModel (Zhipu AI) API Key")).toHaveAttribute("type", "password"); + fill("Credential Name:", "bigmodel-key"); + await user.click(screen.getByRole("button", { name: "Add Credential" })); + expect(await screen.findByText("Required")).toBeInTheDocument(); + expect(onSubmit).not.toHaveBeenCalled(); + }); + + it("saves the key and an explicit base for BigModel", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal(); + await chooseProvider(user, Providers.BIGMODEL); + await screen.findByLabelText("BigModel (Zhipu AI) API Key"); + fill("Credential Name:", "bigmodel-key"); + fill("BigModel (Zhipu AI) API Key", "synthetic-bigmodel-key"); + fill("API Base", "https://bigmodel-proxy.example"); + await user.click(screen.getByRole("button", { name: "Add Credential" })); + expect(onSubmit).toHaveBeenCalledWith( + { + credential_name: "bigmodel-key", + custom_llm_provider: "BIGMODEL", + credential_values: { api_key: "synthetic-bigmodel-key", api_base: "https://bigmodel-proxy.example" }, + }, + [], + ); + }); +}); + describe("CredentialModal with Anthropic workload identity federation", () => { it("deletes stored exchange fields when switching to a static token", async () => { const user = userEvent.setup(); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 0ebd2cdff17..51a248daafa 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -20,6 +20,18 @@ const BUNDLED_MODEL_MAP: Record = JSON.pa ); describe("provider_info_helpers", () => { + it("resolves BigModel metadata, name, logo and model placeholder", () => { + const metadata = PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "BIGMODEL"); + expect(provider_map.BIGMODEL).toBe(metadata?.litellm_provider); + expect(getProviderLogoAndName("bigmodel")).toEqual({ + displayName: "BigModel (Zhipu AI)", + logo: providerLogoMap[Providers.BIGMODEL], + }); + expect(getProviderLogoAndName("BIGMODEL")).toEqual(getProviderLogoAndName("bigmodel")); + expect(providerLogoMap[Providers.BIGMODEL]).toContain("bigmodel"); + expect(getPlaceholder(Providers.BIGMODEL)).toMatch(/^bigmodel\//); + }); + it("maps Microsoft 365 Copilot to its chat model provider and placeholder", () => { expect(provider_map.MICROSOFT_365_COPILOT).toBe("microsoft_365_copilot"); expect(getPlaceholder(Providers.MICROSOFT_365_COPILOT)).toBe("microsoft_365_copilot/chat"); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 2541aa45050..6bda9b4fccb 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -6,6 +6,7 @@ import anthropicLogo from "../../public/assets/logos/anthropic.svg"; import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png"; import basetenLogo from "../../public/assets/logos/baseten.svg"; import bedrockLogo from "../../public/assets/logos/bedrock.svg"; +import bigmodelLogo from "../../public/assets/logos/bigmodel.png"; import cerebrasLogo from "../../public/assets/logos/cerebras.svg"; import cloudflareLogo from "../../public/assets/logos/cloudflare.svg"; import cohereLogo from "../../public/assets/logos/cohere.svg"; @@ -197,6 +198,7 @@ export enum Providers { xAI = "xAI", XINFERENCE = "Xinference", ZAI = "Z.AI (Zhipu AI)", + BIGMODEL = "BigModel (Zhipu AI)", } export const provider_map: Record = { @@ -325,6 +327,7 @@ export const provider_map: Record = { xAI: "xai", XINFERENCE: "xinference", ZAI: "zai", + BIGMODEL: "bigmodel", }; const standaloneSubproviderSlugs = new Set(["bedrock_mantle"]); @@ -333,6 +336,7 @@ export const resolveLitellmProviderSlug = (provider: string): string => provider_map[provider] ?? provider.toLowerCase(); export const providerLogoMap: Partial> = { + [Providers.BIGMODEL]: bigmodelLogo.src, [Providers.A2A_Agent]: a2aAgentLogo.src, [Providers.AI21]: ai21Logo.src, [Providers.AI21_CHAT]: ai21Logo.src, @@ -497,6 +501,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Voyage]: "voyage/", [Providers.WATSONX]: "watsonx/ibm/granite-3-3-8b-instruct", [Providers.ZAI]: "zai/glm-4.5", + [Providers.BIGMODEL]: "bigmodel/glm-5.3", }; export const getPlaceholder = (selectedProvider: string): string => { From e28b2dc4a8ef76a484310cc211d6a068944a8dd8 Mon Sep 17 00:00:00 2001 From: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:11:51 +0800 Subject: [PATCH 02/10] test(providers): make BigModel unit tests a package Co-authored-by: Bytechoreographer --- tests/unit/llms/bigmodel/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit/llms/bigmodel/__init__.py diff --git a/tests/unit/llms/bigmodel/__init__.py b/tests/unit/llms/bigmodel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From fb5c1fd99888f086f430fae9536cb289ba33d65c Mon Sep 17 00:00:00 2001 From: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:21:25 +0800 Subject: [PATCH 03/10] docs(providers): clarify BigModel parameter interface Co-authored-by: Bytechoreographer --- litellm/llms/bigmodel/chat/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py index 1f721d84dd6..6f6914cbdb8 100644 --- a/litellm/llms/bigmodel/chat/transformation.py +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -29,7 +29,7 @@ class BigModelChatConfig(ZAIChatConfig): def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) - def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited interface requires a list return list(_SUPPORTED_PARAMS) def map_openai_params( From 2bd6cc46b3572f6c4e479c375c308a2a6931c773 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:20:43 -0700 Subject: [PATCH 04/10] fix(bigmodel): keep the Coding Plan base, surface Responses auth errors, and scope routes Chat Completions keeps a GLM Coding Plan api_base. Responses raises the status from BigModel's success=false body instead of returning an empty 200, and a JSON body answering a stream request is replayed as a one-shot stream. Messages streams keep ping and [DONE] lines. bigmodel leaves openai_compatible_providers so speech, image, and transcription routes send it no request --- litellm/constants.py | 1 - litellm/llms/bigmodel/common_utils.py | 12 ++- .../llms/bigmodel/messages/transformation.py | 22 +++- .../llms/bigmodel/responses/transformation.py | 14 ++- litellm/llms/custom_httpx/llm_http_handler.py | 19 +++- litellm/main.py | 1 + litellm/utils.py | 1 + .../llms/bigmodel/test_bigmodel_provider.py | 102 ++++++++++++++++-- 8 files changed, 151 insertions(+), 21 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 84ee4a3fca9..8cc51d0ab5e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1084,7 +1084,6 @@ openai_compatible_providers: Final[list[str]] = [ "xinference", "xai", "zai", - "bigmodel", "together_ai", "fireworks_ai", "empower", diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py index d17f478ed29..deb9e54d78b 100644 --- a/litellm/llms/bigmodel/common_utils.py +++ b/litellm/llms/bigmodel/common_utils.py @@ -5,7 +5,10 @@ from urllib.parse import urlsplit, urlunsplit from litellm.secret_managers.main import get_secret_str BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_CODING_CHAT_PATH: Final = "/api/coding/paas/v4" _API_PATHS: Final = ( + "/api/coding/paas/v4/chat/completions", + _CODING_CHAT_PATH, "/api/paas/v4/chat/completions", "/api/anthropic/v1/messages", "/api/anthropic/v1", @@ -32,7 +35,10 @@ def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses" resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE parsed: Final = urlsplit(resolved_base) path: Final = parsed.path.rstrip("/") - root: Final = next((path.removesuffix(suffix) for suffix in _API_PATHS if path.endswith(suffix)), path) - return urlunsplit( - (parsed.scheme, parsed.netloc, f"{root}{_ENDPOINT_PATHS[endpoint]}", parsed.query, parsed.fragment) + suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "") + endpoint_path: Final = ( + _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint] + ) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment) ) diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py index 2cac2d3b899..041adf49374 100644 --- a/litellm/llms/bigmodel/messages/transformation.py +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -4,7 +4,7 @@ from typing import Final import httpx from openai._streaming import ServerSentEvent, SSEDecoder -from pydantic import JsonValue, TypeAdapter +from pydantic import JsonValue, TypeAdapter, ValidationError from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.litellm_logging import Logging @@ -45,20 +45,34 @@ class _BigModelMessagesStream(httpx.AsyncByteStream): await self._response.aclose() -def _normalize_event(event: ServerSentEvent) -> bytes: - body: Final = _JSON_OBJECT.validate_json(event.data) +def _json_object(data: str) -> Mapping[str, JsonValue] | None: + try: + return _JSON_OBJECT.validate_json(data) + except ValidationError: + return None + + +def _normalized_data(data: str) -> str: + body: Final = _json_object(data) + if body is None: + return data message: Final = body.get("message") normalized: Final[JsonValue] = ( {**body, "message": _normalize_message(message)} if body.get("type") == "message_start" and isinstance(message, dict) else _normalize_message(body) ) + return json.dumps(normalized) + + +def _normalize_event(event: ServerSentEvent) -> bytes: metadata: Final = "".join( f"{name}: {value}\n" for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) if value is not None ) - return f"{metadata}data: {json.dumps(normalized)}\n\n".encode() + data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n")) + return f"{metadata}{data_lines}\n".encode() class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py index a61f006371e..6812506729d 100644 --- a/litellm/llms/bigmodel/responses/transformation.py +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -7,6 +7,7 @@ from typing_extensions import Never from litellm.exceptions import AuthenticationError, UnsupportedParamsError from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ( ResponseInputParam, @@ -42,6 +43,10 @@ _SUPPORTED_PARAMS: Final = ( ) +def _envelope_status_code(code: JsonValue) -> int: + return code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code <= 599 else 500 + + def _normalize_output_item(item: JsonValue) -> JsonValue: if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): return item @@ -141,7 +146,14 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_response_api_response( self, model: str, raw_response: httpx.Response, logging_obj: Logging ) -> ResponsesAPIResponse: - normalized: Final = _normalize_response(_JSON_OBJECT.validate_json(raw_response.content)) + body: Final = _JSON_OBJECT.validate_json(raw_response.content) + if body.get("success") is False: + raise BaseLLMException( + message=str(body.get("msg") or raw_response.text), + status_code=_envelope_status_code(body.get("code")), + headers=raw_response.headers, + ) + normalized: Final = _normalize_response(body) normalized_response: Final = httpx.Response( status_code=raw_response.status_code, headers={ diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ef009e70137..afc71f681dd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -365,6 +365,19 @@ class _SignedBodyKwargs(TypedDict, total=False): json: ReadOnly[dict[str, object]] +def _is_json_content(response: httpx.Response) -> bool: + content_types: Final = httpx.Headers(response.headers).get_list("content-type") + media_types: Final = (value.split(";")[0].strip().lower() for value in content_types) + return "application/json" in media_types + + +def _is_json_document(content: bytes) -> bool: + try: + return isinstance(json.loads(content), dict) + except ValueError: + return False + + def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs: if signed_body is not None: return {"data": signed_body} @@ -3022,7 +3035,7 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True: + if fake_stream is True or (_is_json_content(response) and _is_json_document(response.read())): return MockResponsesAPIStreamingIterator( response=response, model=model, @@ -3225,7 +3238,9 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: - if fake_stream is True: + if fake_stream is True or ( + _is_json_content(response) and _is_json_document(await response.aread()) + ): return MockResponsesAPIStreamingIterator( response=response, model=model, diff --git a/litellm/main.py b/litellm/main.py index 57d49c43add..d5427e8062a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6280,6 +6280,7 @@ def completion( or custom_llm_provider == "nebius" or custom_llm_provider == "wandb" or custom_llm_provider == "clarifai" + or custom_llm_provider == "bigmodel" or custom_llm_provider in litellm.openai_compatible_providers or JSONProviderRegistry.exists(custom_llm_provider) # JSON-configured providers or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo diff --git a/litellm/utils.py b/litellm/utils.py index 91922210292..a232fd2f96a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5641,6 +5641,7 @@ def add_provider_specific_params_to_optional_params( "openai", "azure", "text-completion-openai", + "bigmodel", *litellm.openai_compatible_providers, ] or JSONProviderRegistry.exists(custom_llm_provider) if calls_through_openai_sdk: diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index f53673830b2..bb48827f535 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -1,6 +1,8 @@ +import contextlib import gzip +import io import json -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from functools import partial from itertools import chain from typing import Final, Literal @@ -100,7 +102,6 @@ def test_provider_resolution_and_all_native_endpoints() -> None: ProviderConfigManager.get_provider_anthropic_messages_config(MODEL, LlmProviders.BIGMODEL), BigModelMessagesConfig, ) - assert "bigmodel" in litellm.openai_compatible_providers assert "bigmodel" in litellm.provider_list assert litellm.get_llm_provider(f"zai/{MODEL}")[3] == "https://api.z.ai/api/paas/v4" @@ -112,6 +113,15 @@ def test_one_base_selects_each_api_without_duplicate_segments(path: str) -> None assert get_bigmodel_url(f"{BASE}{path}", "messages") == f"{BASE}/api/anthropic/v1/messages" +@pytest.mark.parametrize( + "path", ("/api/coding/paas/v4", "/api/coding/paas/v4/", "/api/coding/paas/v4/chat/completions") +) +def test_coding_plan_base_keeps_its_chat_path_and_shares_the_other_apis(path: str) -> None: + assert get_bigmodel_url(f"{BASE}{path}", "chat") == f"{BASE}/api/coding/paas/v4" + assert get_bigmodel_url(f"{BASE}{path}", "responses") == f"{BASE}/api/v1/responses" + assert get_bigmodel_url(f"{BASE}{path}", "messages") == f"{BASE}/api/anthropic/v1/messages" + + def test_custom_base_and_environment_precedence(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("BIGMODEL_API_BASE", "https://env.example/prefix/api/v1") assert get_bigmodel_url(None, "messages") == "https://env.example/prefix/api/anthropic/v1/messages" @@ -295,10 +305,11 @@ async def test_messages_forward_native_tools_thinking_and_usage(web_fetch_reques (("response.completed", "completed"), ("response.incomplete", "incomplete")), ) @pytest.mark.parametrize("include_logprobs", (False, True)) +@pytest.mark.parametrize("content_type", ("text/event-stream", "application/json")) @pytest.mark.asyncio @respx.mock async def test_responses_sse_without_done_preserves_terminal_usage( - terminal_type: str, status: str, include_logprobs: bool + terminal_type: str, status: str, include_logprobs: bool, content_type: str ) -> None: logprobs: Final = [{"token": "Hello", "bytes": [72, 101, 108, 108, 111], "logprob": -0.1, "top_logprobs": []}] events: Final = ( @@ -348,7 +359,7 @@ async def test_responses_sse_without_done_preserves_terminal_usage( {"type": terminal_type, "sequence_number": 5, "response": {**response_payload(), "status": status}}, ) sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) - respx.post(f"{BASE}/api/v1/responses").respond(content=sse, headers={"content-type": "text/event-stream"}) + respx.post(f"{BASE}/api/v1/responses").respond(content=sse, headers={"content-type": content_type}) stream: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) chunks: Final = tuple([chunk async for chunk in stream]) assert chunks[-1].type == terminal_type @@ -392,6 +403,72 @@ def test_missing_bigmodel_key_does_not_use_other_provider_credentials( call() +def _audio_file() -> io.BytesIO: + audio: Final = io.BytesIO(b"RIFF0000WAVEfmt ") + audio.name = "audio.wav" + return audio + + +@pytest.mark.parametrize( + "call", + ( + partial(litellm.speech, model=f"bigmodel/{MODEL}", input="Hello", voice="tongtong"), + partial(litellm.image_generation, model=f"bigmodel/{MODEL}", prompt="Hello"), + lambda: litellm.transcription(model=f"bigmodel/{MODEL}", file=_audio_file()), + ), + ids=("speech", "image_generation", "transcription"), +) +@respx.mock +def test_openai_routes_bigmodel_does_not_serve_send_no_request( + call: Callable[[], object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + upstream: Final = respx.route().mock(return_value=httpx.Response(200, json={})) + with contextlib.suppress(Exception): + call() + assert not upstream.called + + +@pytest.mark.parametrize( + ("code", "error_class", "status_code"), + ((401, litellm.AuthenticationError, 401), (1113, litellm.InternalServerError, 500)), +) +@respx.mock +def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( + code: int, error_class: type[Exception], status_code: int +) -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json={"code": code, "msg": "Token expired", "success": False}) + with pytest.raises(error_class, match="Token expired") as raised: + litellm.responses(model=f"bigmodel/{MODEL}", input="Hello") + assert getattr(raised.value, "status_code", None) == status_code + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_stream_answered_with_a_failure_envelope_raises_its_status() -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json={"code": 401, "msg": "Token expired", "success": False}) + with pytest.raises(litellm.AuthenticationError, match="Token expired"): + await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + with pytest.raises(litellm.AuthenticationError, match="Token expired"): + litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_responses_stream_answered_with_one_json_response_replays_it_as_events(is_async: bool) -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + request: Final = {"model": f"bigmodel/{MODEL}", "input": "Hello", "stream": True} + chunks: Final = ( + tuple([chunk async for chunk in await litellm.aresponses(**request)]) + if is_async + else tuple(litellm.responses(**request)) + ) + assert chunks[-1].type == "response.completed" + assert chunks[-1].response.output_text == "Hello" + assert chunks[-1].response.usage.total_tokens == 30 + + @pytest.mark.parametrize("event_type", ("response.failed", "error")) @pytest.mark.asyncio @respx.mock @@ -633,9 +710,15 @@ async def test_messages_stream_preserves_native_events() -> None: }, {"type": "message_stop"}, ) + native_events: Final = tuple( + f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" for event in events + ) content: Final = ( "id: bigmodel-event\nretry: 1000\n" - + "".join(f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" for event in events) + + native_events[0] + + "event: ping\n\n" + + "".join(native_events[1:]) + + "data: [DONE]\n\n" ).encode() upstream: Final = _ByteChunks(tuple(content[index : index + 7] for index in range(0, len(content), 7))) respx.post(f"{BASE}/api/anthropic/v1/messages").mock( @@ -652,11 +735,8 @@ async def test_messages_stream_preserves_native_events() -> None: stream=True, ) output: Final = tuple([chunk async for chunk in stream]) - parsed: Final = tuple( - json.loads(line.removeprefix("data: ")) - for line in chain.from_iterable(chunk.decode().splitlines() for chunk in output) - if line.startswith("data: ") - ) + lines: Final = tuple(chain.from_iterable(chunk.decode().splitlines() for chunk in output)) + parsed: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines if line.startswith("data: {")) assert tuple(chunk["type"] for chunk in parsed) == tuple(event["type"] for event in events) assert tuple(event["delta"] for event in parsed if event["type"] == "content_block_delta") == ( {"type": "thinking_delta", "thinking": "先思考"}, @@ -668,6 +748,8 @@ async def test_messages_stream_preserves_native_events() -> None: "web_search_requests": 0, "web_fetch_requests": 0, } + assert "event: ping" in lines + assert "data: [DONE]" in lines assert b"id: bigmodel-event\n" in output[0] assert b"retry: 1000\n" in output[0] assert upstream.is_closed From 807a2aecfb97cc40156cc506eb1e03bc250f83ec Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:41:56 -0700 Subject: [PATCH 05/10] fix(responses): replay a JSON body on a stream request only when it has a Content-Length A gateway that labels a chunked SSE stream as application/json kept streaming before the replay, so only a sized body is read in full --- litellm/llms/custom_httpx/llm_http_handler.py | 12 ++--- .../llms/bigmodel/test_bigmodel_provider.py | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index afc71f681dd..0c74748ec7d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -365,10 +365,10 @@ class _SignedBodyKwargs(TypedDict, total=False): json: ReadOnly[dict[str, object]] -def _is_json_content(response: httpx.Response) -> bool: - content_types: Final = httpx.Headers(response.headers).get_list("content-type") - media_types: Final = (value.split(";")[0].strip().lower() for value in content_types) - return "application/json" in media_types +def _is_sized_json_body(response: httpx.Response) -> bool: + headers: Final = httpx.Headers(response.headers) + media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type")) + return "content-length" in headers and "application/json" in media_types def _is_json_document(content: bytes) -> bool: @@ -3035,7 +3035,7 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True or (_is_json_content(response) and _is_json_document(response.read())): + if fake_stream is True or (_is_sized_json_body(response) and _is_json_document(response.read())): return MockResponsesAPIStreamingIterator( response=response, model=model, @@ -3239,7 +3239,7 @@ class BaseLLMHTTPHandler: if caller_requested_stream: if fake_stream is True or ( - _is_json_content(response) and _is_json_document(await response.aread()) + _is_sized_json_body(response) and _is_json_document(await response.aread()) ): return MockResponsesAPIStreamingIterator( response=response, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index bb48827f535..5c485ef0291 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -1,3 +1,4 @@ +import asyncio import contextlib import gzip import io @@ -469,6 +470,51 @@ async def test_responses_stream_answered_with_one_json_response_replays_it_as_ev assert chunks[-1].response.usage.total_tokens == 30 +class _GatedStream(httpx.AsyncByteStream): + def __init__(self, first: bytes, rest: bytes, gate: asyncio.Event) -> None: + self._first = first + self._rest = rest + self._gate = gate + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._first + await self._gate.wait() + yield self._rest + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_stream_labeled_json_without_a_length_yields_events_before_it_ends() -> None: + created: Final = { + "type": "response.created", + "sequence_number": 0, + "response": {**response_payload(), "status": "in_progress", "output": [], "usage": None}, + } + completed: Final = {"type": "response.completed", "sequence_number": 1, "response": response_payload()} + gate: Final = asyncio.Event() + respx.post(f"{BASE}/api/v1/responses").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "application/json"}, + stream=_GatedStream( + f"event: response.created\ndata: {json.dumps(created)}\n\n".encode(), + f"event: response.completed\ndata: {json.dumps(completed)}\n\n".encode(), + gate, + ), + ) + ) + + stream: Final = await asyncio.wait_for( + litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True), timeout=10 + ) + first: Final = await asyncio.wait_for(anext(stream), timeout=10) + gate.set() + rest: Final = tuple([chunk async for chunk in stream]) + assert first.type == "response.created" + assert rest[-1].type == "response.completed" + assert rest[-1].response.output_text == "Hello" + + @pytest.mark.parametrize("event_type", ("response.failed", "error")) @pytest.mark.asyncio @respx.mock From cf201798fca892bee4faf35c9ed782e873b28154 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:18:59 -0700 Subject: [PATCH 06/10] fix(bigmodel): scope the JSON stream replay to BigModel, map its business error codes, and echo requested tools --- examples/bigmodel.md | 2 + .../llms/base_llm/responses/transformation.py | 3 ++ .../llms/bigmodel/responses/transformation.py | 52 ++++++++++++++++--- litellm/llms/custom_httpx/llm_http_handler.py | 10 +++- .../llms/bigmodel/test_bigmodel_provider.py | 28 +++++++++- .../custom_httpx/test_llm_http_handler.py | 13 +++++ 6 files changed, 99 insertions(+), 9 deletions(-) diff --git a/examples/bigmodel.md b/examples/bigmodel.md index 1ae64bcbfc4..1f0bb4cd2af 100644 --- a/examples/bigmodel.md +++ b/examples/bigmodel.md @@ -56,6 +56,8 @@ GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 suppo Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search +Codex sends `include` and `parallel_tool_calls` on every Responses request, and LiteLLM does not forward either to BigModel. Set `drop_params: true` on the deployment so LiteLLM drops them instead of answering 400 + The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 01baeed9bef..7f45c1eba28 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -268,6 +268,9 @@ class BaseResponsesAPIConfig(ABC): """Returns True if litellm should fake a stream for the given model and stream value""" return False + def may_answer_stream_with_json(self) -> bool: + return False + def supports_native_websocket(self) -> bool: """ Returns True if the provider has a native WebSocket endpoint for Responses API. diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py index 6812506729d..0957f57832c 100644 --- a/litellm/llms/bigmodel/responses/transformation.py +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -1,8 +1,9 @@ from collections.abc import Mapping +from types import MappingProxyType from typing import Final import httpx -from pydantic import JsonValue, TypeAdapter +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError from typing_extensions import Never from litellm.exceptions import AuthenticationError, UnsupportedParamsError @@ -43,8 +44,39 @@ _SUPPORTED_PARAMS: Final = ( ) +_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType( + { + **dict.fromkeys((1000, 1001, 1003, 1005), 401), + **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400), + 1220: 403, + **dict.fromkeys( + (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321), + 429, + ), + } +) + + +class _RequestedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + tools: JsonValue = None + tool_choice: JsonValue = None + + +def _requested_tools(logging_obj: Logging) -> _RequestedTools: + try: + return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {}) + except ValidationError: + return _RequestedTools() + + def _envelope_status_code(code: JsonValue) -> int: - return code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code <= 599 else 500 + if not isinstance(code, int) or isinstance(code, bool): + return 500 + if 400 <= code <= 599: + return code + return _STATUS_BY_BUSINESS_CODE.get(code, 500) def _normalize_output_item(item: JsonValue) -> JsonValue: @@ -59,6 +91,7 @@ def _normalize_output_item(item: JsonValue) -> JsonValue: def _normalize_response( body: Mapping[str, JsonValue], + requested: _RequestedTools, ) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary usage: Final = body.get("usage") output: Final = body.get("output") @@ -77,8 +110,8 @@ def _normalize_response( normalized: Final[JsonValue] = { "parallel_tool_calls": False, **body, - "tools": [] if tools is None else tools, - "tool_choice": "auto" if tool_choice is None else tool_choice, + "tools": (requested.tools or []) if tools is None else tools, + "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, **({"usage": normalized_usage} if "usage" in body else {}), **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), } @@ -115,6 +148,9 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): ) -> bool: return False + def may_answer_stream_with_json(self) -> bool: + return True + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list return list(_SUPPORTED_PARAMS) @@ -153,7 +189,7 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): status_code=_envelope_status_code(body.get("code")), headers=raw_response.headers, ) - normalized: Final = _normalize_response(body) + normalized: Final = _normalize_response(body, _requested_tools(logging_obj)) normalized_response: Final = httpx.Response( status_code=raw_response.status_code, headers={ @@ -179,7 +215,11 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): item: Final = chunk.get("item") normalized: Final[JsonValue] = { **chunk, - **({"response": _normalize_response(response)} if isinstance(response, dict) else {}), + **( + {"response": _normalize_response(response, _requested_tools(logging_obj))} + if isinstance(response, dict) + else {} + ), **({"item": _normalize_output_item(item)} if "item" in chunk else {}), } event_model: Final = self.get_event_model_class(str(normalized.get("type"))) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 0c74748ec7d..26a0d4df97f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3035,7 +3035,11 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True or (_is_sized_json_body(response) and _is_json_document(response.read())): + if fake_stream is True or ( + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(response.read()) + ): return MockResponsesAPIStreamingIterator( response=response, model=model, @@ -3239,7 +3243,9 @@ class BaseLLMHTTPHandler: if caller_requested_stream: if fake_stream is True or ( - _is_sized_json_body(response) and _is_json_document(await response.aread()) + responses_api_provider_config.may_answer_stream_with_json() + and _is_sized_json_body(response) + and _is_json_document(await response.aread()) ): return MockResponsesAPIStreamingIterator( response=response, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index 5c485ef0291..a9ecbe66118 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -432,7 +432,12 @@ def test_openai_routes_bigmodel_does_not_serve_send_no_request( @pytest.mark.parametrize( ("code", "error_class", "status_code"), - ((401, litellm.AuthenticationError, 401), (1113, litellm.InternalServerError, 500)), + ( + (401, litellm.AuthenticationError, 401), + (1113, litellm.RateLimitError, 429), + (1301, litellm.BadRequestError, 400), + (1200, litellm.InternalServerError, 500), + ), ) @respx.mock def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( @@ -444,6 +449,27 @@ def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( assert getattr(raised.value, "status_code", None) == status_code +@pytest.mark.parametrize("stream", (False, True)) +@respx.mock +def test_responses_without_tools_in_the_answer_echo_the_requested_tools(stream: bool) -> None: + tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object", "properties": {}}}] + completed: Final = {"type": "response.completed", "sequence_number": 0, "response": response_payload()} + respx.post(f"{BASE}/api/v1/responses").respond( + **( + {"content": f"event: response.completed\ndata: {json.dumps(completed)}\n\n"} + if stream + else {"json": response_payload()} + ) + ) + answer: Final = litellm.responses( + model=f"bigmodel/{MODEL}", input="Hello", tools=tools, tool_choice="required", stream=stream + ) + response: Final = tuple(answer)[-1].response if stream else answer + validated: Final = OpenAIResponse.model_validate(response.model_dump()) + assert validated.tools[0].name == "weather" + assert validated.tool_choice == "required" + + @pytest.mark.asyncio @respx.mock async def test_responses_stream_answered_with_a_failure_envelope_raises_its_status() -> None: diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 297e64629a8..2590567726e 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5695,3 +5695,16 @@ async def test_non_chatgpt_responses_preserves_caller_headers_after_validation( await client.close() else: client.client.close() + + +@pytest.mark.asyncio +async def test_a_stream_answered_with_one_json_body_is_not_replayed_for_a_provider_that_does_not_opt_in( + httpx_transport: None, +) -> None: + with respx.mock(assert_all_called=True) as router: + router.post("https://api.openai.com/v1/responses").respond(json={"error": {"message": "quota exceeded"}}) + stream: Final = await litellm.aresponses( + model="openai/gpt-5.5", input="Hello", api_key="fake-key", stream=True + ) + with pytest.raises(litellm.exceptions.MidStreamFallbackError, match="before any terminal event"): + _ = tuple([event async for event in stream]) From 2a247465ff06a43fc17ac834ce90d2d34924736f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:56:57 -0700 Subject: [PATCH 07/10] fix(bigmodel): forward Codex's include and parallel_tool_calls, keep the OpenAI org header off BigModel, and pass unchanged Messages stream events through --- examples/bigmodel.md | 2 - .../llms/bigmodel/messages/transformation.py | 2 +- .../llms/bigmodel/responses/transformation.py | 8 +++- litellm/main.py | 3 +- .../llms/bigmodel/test_bigmodel_provider.py | 44 ++++++++++++++++--- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/examples/bigmodel.md b/examples/bigmodel.md index 1f0bb4cd2af..1ae64bcbfc4 100644 --- a/examples/bigmodel.md +++ b/examples/bigmodel.md @@ -56,8 +56,6 @@ GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 suppo Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search -Codex sends `include` and `parallel_tool_calls` on every Responses request, and LiteLLM does not forward either to BigModel. Set `drop_params: true` on the deployment so LiteLLM drops them instead of answering 400 - The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py index 041adf49374..7af6fc6f6c3 100644 --- a/litellm/llms/bigmodel/messages/transformation.py +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -62,7 +62,7 @@ def _normalized_data(data: str) -> str: if body.get("type") == "message_start" and isinstance(message, dict) else _normalize_message(body) ) - return json.dumps(normalized) + return data if normalized == body else json.dumps(normalized, ensure_ascii=False) def _normalize_event(event: ServerSentEvent) -> bytes: diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py index 0957f57832c..cb3ec1050ff 100644 --- a/litellm/llms/bigmodel/responses/transformation.py +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -32,6 +32,8 @@ _SUPPORTED_PARAMS: Final = ( "max_output_tokens", "tools", "tool_choice", + "parallel_tool_calls", + "include", "reasoning", "text", "prompt_cache_key", @@ -62,6 +64,7 @@ class _RequestedTools(BaseModel): tools: JsonValue = None tool_choice: JsonValue = None + parallel_tool_calls: JsonValue = None def _requested_tools(logging_obj: Logging) -> _RequestedTools: @@ -97,6 +100,7 @@ def _normalize_response( output: Final = body.get("output") tools: Final = body.get("tools") tool_choice: Final = body.get("tool_choice") + parallel_tool_calls: Final = body.get("parallel_tool_calls") input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None normalized_usage: Final = ( @@ -108,8 +112,10 @@ def _normalize_response( else usage ) normalized: Final[JsonValue] = { - "parallel_tool_calls": False, **body, + "parallel_tool_calls": ( + requested.parallel_tool_calls is True if parallel_tool_calls is None else parallel_tool_calls + ), "tools": (requested.tools or []) if tools is None else tools, "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, **({"usage": normalized_usage} if "usage" in body else {}), diff --git a/litellm/main.py b/litellm/main.py index d5427e8062a..626313c5b58 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2672,8 +2672,7 @@ def _complete_custom_openai( ) organization = ( organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") + or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION")) or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) openai.organization = organization diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index a9ecbe66118..f36604ca9a2 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -180,6 +180,23 @@ async def test_chat_uses_native_endpoint_and_maps_token_limit(is_async: bool) -> assert json.loads(route.calls.last.request.content)["thinking"] == {"type": "enabled"} +@respx.mock +def test_chat_does_not_send_the_openai_organization_to_bigmodel(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_ORGANIZATION", "org-openai-only") + route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "object": "chat.completion", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + _ = litellm.completion(model=f"bigmodel/{MODEL}", messages=[{"role": "user", "content": "Hello"}]) + assert "openai-organization" not in route.calls.last.request.headers + + @pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.asyncio @respx.mock @@ -451,23 +468,37 @@ def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( @pytest.mark.parametrize("stream", (False, True)) @respx.mock -def test_responses_without_tools_in_the_answer_echo_the_requested_tools(stream: bool) -> None: +def test_responses_forward_codex_tool_params_and_echo_them_when_the_answer_omits_them(stream: bool) -> None: tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object", "properties": {}}}] - completed: Final = {"type": "response.completed", "sequence_number": 0, "response": response_payload()} - respx.post(f"{BASE}/api/v1/responses").respond( + completed: Final = { + "type": "response.completed", + "sequence_number": 0, + "response": {**response_payload(), "parallel_tool_calls": None}, + } + route: Final = respx.post(f"{BASE}/api/v1/responses").respond( **( {"content": f"event: response.completed\ndata: {json.dumps(completed)}\n\n"} if stream - else {"json": response_payload()} + else {"json": {**response_payload(), "parallel_tool_calls": None}} ) ) answer: Final = litellm.responses( - model=f"bigmodel/{MODEL}", input="Hello", tools=tools, tool_choice="required", stream=stream + model=f"bigmodel/{MODEL}", + input="Hello", + tools=tools, + tool_choice="required", + parallel_tool_calls=True, + include=["reasoning.encrypted_content"], + stream=stream, ) response: Final = tuple(answer)[-1].response if stream else answer validated: Final = OpenAIResponse.model_validate(response.model_dump()) + sent: Final = json.loads(route.calls.last.request.content) + assert sent["parallel_tool_calls"] is True + assert sent["include"] == ["reasoning.encrypted_content"] assert validated.tools[0].name == "weather" assert validated.tool_choice == "required" + assert validated.parallel_tool_calls is True @pytest.mark.asyncio @@ -820,6 +851,9 @@ async def test_messages_stream_preserves_native_events() -> None: "web_search_requests": 0, "web_fetch_requests": 0, } + assert tuple(line for line in lines if line.startswith("data: {"))[1:] == tuple( + f"data: {json.dumps(event, ensure_ascii=False)}" for event in events[1:] + ) assert "event: ping" in lines assert "data: [DONE]" in lines assert b"id: bigmodel-event\n" in output[0] From 4c6bdbb2ffb3abbc0867eb080f6c61e35a97bfd6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:02:36 -0700 Subject: [PATCH 08/10] fix(bigmodel): map disabled thinking to low effort, keep Messages temperature, raise context window and content policy errors, and default parallel_tool_calls to true --- examples/bigmodel.md | 2 + .../exception_mapping_utils.py | 3 + litellm/llms/bigmodel/chat/transformation.py | 14 +- litellm/llms/bigmodel/common_utils.py | 48 +++++++ .../llms/bigmodel/messages/transformation.py | 44 +++++- .../llms/bigmodel/responses/transformation.py | 11 +- .../llms/bigmodel/test_bigmodel_provider.py | 126 +++++++++++++++++- 7 files changed, 235 insertions(+), 13 deletions(-) diff --git a/examples/bigmodel.md b/examples/bigmodel.md index 1ae64bcbfc4..88a69796ebe 100644 --- a/examples/bigmodel.md +++ b/examples/bigmodel.md @@ -21,6 +21,8 @@ One model configuration supports all three native APIs. The provider selects the `api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix +A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above + The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` ```python diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 1c816b22ef1..69c869a814f 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -2758,6 +2759,8 @@ def exception_type( exception_provider=exception_provider, extra_information=extra_information, ) + if custom_llm_provider == "bigmodel": + raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception) # pyright: ignore[reportUnknownArgumentType] # exception_type takes untyped parameters if custom_llm_provider == "openrouter": _map_openrouter_exception( model=model, diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py index 6f6914cbdb8..12d5e912c15 100644 --- a/litellm/llms/bigmodel/chat/transformation.py +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -3,7 +3,7 @@ from typing import Final from litellm.llms.zai.chat.transformation import ZAIChatConfig -from ..common_utils import get_bigmodel_api_key, get_bigmodel_url +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking _SUPPORTED_PARAMS: Final = ( "max_tokens", @@ -49,7 +49,11 @@ class BigModelChatConfig(ZAIChatConfig): } if "thinking" not in non_default_params: return mapped - return { - **mapped, - "extra_body": {"thinking": non_default_params["thinking"]}, - } + thinking: Final = non_default_params["thinking"] + if is_unsupported_disabled_thinking(model, thinking): + return { + **mapped, + "reasoning_effort": mapped.get("reasoning_effort", "low"), + "extra_body": {"thinking": {"type": "enabled"}}, + } + return {**mapped, "extra_body": {"thinking": thinking}} diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py index deb9e54d78b..0c9b6da5c8f 100644 --- a/litellm/llms/bigmodel/common_utils.py +++ b/litellm/llms/bigmodel/common_utils.py @@ -1,10 +1,20 @@ +import re +from collections.abc import Mapping from types import MappingProxyType from typing import Final, Literal from urllib.parse import urlsplit, urlunsplit +import httpx +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""") _CODING_CHAT_PATH: Final = "/api/coding/paas/v4" _API_PATHS: Final = ( "/api/coding/paas/v4/chat/completions", @@ -42,3 +52,41 @@ def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses" return urlunsplit( (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment) ) + + +def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]: + try: + return _JSON_OBJECT.validate_json(response.content) + except ValidationError: + raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers) + + +def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool: + return thinking == {"type": "disabled"} and litellm.utils.supports_factory( + model=model, custom_llm_provider="bigmodel", key="thinking_always_on" + ) + + +def bigmodel_business_error( + code: JsonValue, message: str, model: str, response: httpx.Response | None +) -> ContextWindowExceededError | ContentPolicyViolationError | None: + match code: + case 1261 | "1261": + return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response) + case 1301 | "1301": + return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response) + case _: + return None + + +def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None: + found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str) + response: Final = getattr(original_exception, "response", None) + error: Final = bigmodel_business_error( + found.group(1) if found else None, + error_str, + str(model), + response if isinstance(response, httpx.Response) else None, + ) + if error is not None: + raise error diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py index 7af6fc6f6c3..8ea9f12edf9 100644 --- a/litellm/llms/bigmodel/messages/transformation.py +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -13,10 +13,27 @@ from litellm.llms.openai_like.messages.transformation import ( OpenAILikeAnthropicMessagesConfig, ) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +from litellm.types.router import GenericLiteLLMParams -from ..common_utils import get_bigmodel_api_key, get_bigmodel_url +from ..common_utils import ( + bigmodel_json_body, + get_bigmodel_api_key, + get_bigmodel_url, + is_unsupported_disabled_thinking, +) _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object]) + + +def _with_lowest_effort_thinking( + params: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: the parent transform takes a request dict + requested_output_config: Final = params.get("output_config") + output_config: Final = ( + _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {} + ) + return {**params, "thinking": {"type": "adaptive"}, "output_config": {"effort": "low", **output_config}} def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response @@ -86,12 +103,29 @@ class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): def should_strip_billing_metadata(self) -> bool: return True + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict[str, object]], # mutable-ok: matches dict-typed base signature + anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches dict-typed base signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: matches dict-typed base signature + ) -> dict[str, object]: # mutable-ok: matches dict-typed base signature + request_params: Final = ( + _with_lowest_effort_thinking(anthropic_messages_optional_request_params) + if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking")) + else anthropic_messages_optional_request_params + ) + return super().transform_anthropic_messages_request( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the parent transform returns a bare dict + model, messages, request_params, litellm_params, headers + ) + def transform_anthropic_messages_response( self, model: str, raw_response: httpx.Response, logging_obj: Logging ) -> AnthropicMessagesResponse: normalized: Final = httpx.Response( raw_response.status_code, - json=_normalize_message(_JSON_OBJECT.validate_json(raw_response.content)), + json=_normalize_message(bigmodel_json_body(raw_response)), request=raw_response.request, ) return super().transform_anthropic_messages_response(model, normalized, logging_obj) @@ -129,6 +163,12 @@ class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): ) -> None: return None + @staticmethod + def _drop_incompatible_temperature_for_thinking( + model: str, optional_params: Mapping[str, object], custom_llm_provider: str + ) -> None: + return None + def validate_anthropic_messages_environment( self, headers: Mapping[str, str], diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py index cb3ec1050ff..80c53682d16 100644 --- a/litellm/llms/bigmodel/responses/transformation.py +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..common_utils import get_bigmodel_api_key, get_bigmodel_url +from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _SUPPORTED_PARAMS: Final = ( @@ -114,7 +114,7 @@ def _normalize_response( normalized: Final[JsonValue] = { **body, "parallel_tool_calls": ( - requested.parallel_tool_calls is True if parallel_tool_calls is None else parallel_tool_calls + requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls ), "tools": (requested.tools or []) if tools is None else tools, "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice, @@ -188,10 +188,11 @@ class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): def transform_response_api_response( self, model: str, raw_response: httpx.Response, logging_obj: Logging ) -> ResponsesAPIResponse: - body: Final = _JSON_OBJECT.validate_json(raw_response.content) + body: Final = bigmodel_json_body(raw_response) if body.get("success") is False: - raise BaseLLMException( - message=str(body.get("msg") or raw_response.text), + message: Final = str(body.get("msg") or raw_response.text) + raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException( + message=message, status_code=_envelope_status_code(body.get("code")), headers=raw_response.headers, ) diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index f36604ca9a2..7d2eb8b3ec0 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -266,6 +266,81 @@ def test_messages_preserves_native_reasoning_params(reasoning_params: Mapping[st assert {key: payload.get(key) for key in reasoning_params} == reasoning_params +@pytest.mark.parametrize( + ("model", "extra", "expected"), + ( + (MODEL, {}, {"thinking": {"type": "enabled"}, "reasoning_effort": "low"}), + (MODEL, {"reasoning_effort": "high"}, {"thinking": {"type": "enabled"}, "reasoning_effort": "high"}), + ("glm-4.7", {}, {"thinking": {"type": "disabled"}, "reasoning_effort": None}), + ), +) +@respx.mock +def test_chat_asks_an_always_thinking_model_for_its_lowest_effort_when_thinking_is_disabled( + model: str, extra: Mapping[str, JsonValue], expected: Mapping[str, JsonValue] +) -> None: + route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + } + ) + litellm.completion( + model=f"bigmodel/{model}", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + **extra, + ) + body: Final = json.loads(route.calls.last.request.content) + assert {key: body.get(key) for key in expected} == expected + + +@pytest.mark.parametrize( + ("model", "extra", "expected"), + ( + (MODEL, {}, {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}), + ( + MODEL, + {"output_config": {"effort": "high"}}, + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}, + ), + ("glm-4.7", {}, {"thinking": {"type": "disabled"}, "output_config": None}), + ), +) +def test_messages_asks_an_always_thinking_model_for_its_lowest_effort_when_thinking_is_disabled( + model: str, extra: Mapping[str, JsonValue], expected: Mapping[str, JsonValue] +) -> None: + payload: Final = BigModelMessagesConfig().transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 64, "thinking": {"type": "disabled"}, **extra}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert {key: payload.get(key) for key in expected} == expected + + +@pytest.mark.parametrize( + "reasoning_params", + ({"thinking": {"type": "enabled", "budget_tokens": 1024}}, {"output_config": {"effort": "low"}}), +) +@pytest.mark.parametrize("temperature", (0, 0.2)) +def test_messages_keeps_the_callers_temperature_next_to_thinking( + reasoning_params: Mapping[str, JsonValue], temperature: float +) -> None: + payload: Final = BigModelMessagesConfig().transform_anthropic_messages_request( + model=MODEL, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 2048, "temperature": temperature, **reasoning_params}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert payload.get("temperature") == temperature + + @pytest.mark.asyncio @respx.mock @pytest.mark.parametrize("web_fetch_requests", (None, 2)) @@ -385,7 +460,7 @@ async def test_responses_sse_without_done_preserves_terminal_usage( assert chunks[-1].response.usage.total_tokens == 30 assert chunks[-1].response.output_text == "Hello" OpenAIResponse.model_validate(chunks[-1].response.model_dump()) - assert chunks[0].response.parallel_tool_calls is False + assert chunks[0].response.parallel_tool_calls is True assert chunks[-2].logprobs == (logprobs if include_logprobs else []) assert tuple(chunk.delta for chunk in chunks if chunk.type == "response.output_text.delta") == ("Hello",) assert tuple(chunk.part.text for chunk in chunks if chunk.type == "response.content_part.done") == ( @@ -466,6 +541,55 @@ def test_responses_failure_envelope_raises_its_status_instead_of_a_parse_error( assert getattr(raised.value, "status_code", None) == status_code +@pytest.mark.parametrize( + ("code", "error_class"), + ((1261, litellm.ContextWindowExceededError), (1301, litellm.ContentPolicyViolationError)), +) +@pytest.mark.parametrize("route", ("chat", "responses")) +@respx.mock +def test_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch( + code: int, error_class: type[Exception], route: Literal["chat", "responses"] +) -> None: + if route == "chat": + respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + status_code=400, json={"error": {"code": str(code), "message": "rejected by BigModel"}} + ) + else: + respx.post(f"{BASE}/api/v1/responses").respond( + json={"code": code, "msg": "rejected by BigModel", "success": False} + ) + call: Final = ( + (lambda: litellm.completion(model=f"bigmodel/{MODEL}", messages=[{"role": "user", "content": "Hello"}])) + if route == "chat" + else (lambda: litellm.responses(model=f"bigmodel/{MODEL}", input="Hello")) + ) + with pytest.raises(error_class, match="rejected by BigModel") as raised: + call() + assert raised.value.status_code == 400 + + +@pytest.mark.parametrize("route", ("responses", "messages")) +@pytest.mark.asyncio +@respx.mock +async def test_a_non_json_answer_surfaces_its_body_text(route: Literal["responses", "messages"]) -> None: + path: Final = "/api/v1/responses" if route == "responses" else "/api/anthropic/v1/messages" + respx.post(f"{BASE}{path}").respond( + content=b"gateway maintenance", headers={"content-type": "text/html"} + ) + call: Final = ( + (lambda: litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello")) + if route == "responses" + else ( + lambda: litellm.anthropic.messages.acreate( + model=f"bigmodel/{MODEL}", max_tokens=16, messages=[{"role": "user", "content": "Hello"}] + ) + ) + ) + with pytest.raises(Exception, match="gateway maintenance") as raised: + await call() + assert "validation error" not in str(raised.value) + + @pytest.mark.parametrize("stream", (False, True)) @respx.mock def test_responses_forward_codex_tool_params_and_echo_them_when_the_answer_omits_them(stream: bool) -> None: From 7079bab988a5be700b3158577c18422e3f11cd06 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:08:59 -0700 Subject: [PATCH 09/10] fix(bigmodel): keep the caller's reasoning_effort when Messages rewrites disabled thinking --- litellm/llms/bigmodel/messages/transformation.py | 11 +++++++---- tests/unit/llms/bigmodel/test_bigmodel_provider.py | 5 +++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py index 8ea9f12edf9..82ce59631f1 100644 --- a/litellm/llms/bigmodel/messages/transformation.py +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -30,10 +30,13 @@ def _with_lowest_effort_thinking( params: Mapping[str, object], ) -> dict[str, object]: # mutable-ok: the parent transform takes a request dict requested_output_config: Final = params.get("output_config") - output_config: Final = ( - _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {} - ) - return {**params, "thinking": {"type": "adaptive"}, "output_config": {"effort": "low", **output_config}} + output_config: Final = { + **({} if "reasoning_effort" in params else {"effort": "low"}), + **( + _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {} + ), + } + return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})} def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index 7d2eb8b3ec0..60754cebe9f 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -307,6 +307,11 @@ def test_chat_asks_an_always_thinking_model_for_its_lowest_effort_when_thinking_ {"output_config": {"effort": "high"}}, {"thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}, ), + ( + MODEL, + {"reasoning_effort": "high"}, + {"thinking": {"type": "adaptive"}, "reasoning_effort": "high", "output_config": None}, + ), ("glm-4.7", {}, {"thinking": {"type": "disabled"}, "output_config": None}), ), ) From 9fb6a008299bf8c4b54542f1249c592a8f57fc0b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:20:56 -0700 Subject: [PATCH 10/10] fix(bigmodel): keep context-window and content-policy errors on streamed Responses replays --- litellm/llms/custom_httpx/llm_http_handler.py | 62 +++++++++++-------- .../llms/bigmodel/test_bigmodel_provider.py | 24 +++++++ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7ef54fd7ba6..4e1813e2aee 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3037,6 +3037,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = sync_httpx_client.post( @@ -3047,12 +3058,14 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True or ( + if fake_stream is True: + return replay_as_stream(response=response) + if not ( responses_api_provider_config.may_answer_stream_with_json() and _is_sized_json_body(response) and _is_json_document(response.read()) ): - return MockResponsesAPIStreamingIterator( + return SyncResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3062,17 +3075,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return SyncResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) response.read() else: response = sync_httpx_client.post( @@ -3087,6 +3089,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3242,6 +3247,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = await async_httpx_client.post( @@ -3254,12 +3270,14 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: - if fake_stream is True or ( + if fake_stream is True: + return replay_as_stream(response=response) + if not ( responses_api_provider_config.may_answer_stream_with_json() and _is_sized_json_body(response) and _is_json_document(await response.aread()) ): - return MockResponsesAPIStreamingIterator( + return ResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3269,17 +3287,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return ResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) await response.aread() else: response = await async_httpx_client.post( @@ -3296,6 +3303,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index 60754cebe9f..cdf98c8f9a4 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -573,6 +573,30 @@ def test_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch( assert raised.value.status_code == 400 +@pytest.mark.parametrize( + ("code", "error_class"), + ((1261, litellm.ContextWindowExceededError), (1301, litellm.ContentPolicyViolationError)), +) +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_streamed_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch( + code: int, error_class: type[Exception], is_async: bool +) -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json={"code": code, "msg": "rejected by BigModel", "success": False}) + params: Final = {"model": f"bigmodel/{MODEL}", "input": "Hello", "stream": True} + + async def call() -> None: + if is_async: + await litellm.aresponses(**params) + return + litellm.responses(**params) + + with pytest.raises(error_class, match="rejected by BigModel") as raised: + await call() + assert raised.value.status_code == 400 + + @pytest.mark.parametrize("route", ("responses", "messages")) @pytest.mark.asyncio @respx.mock