From 960d891522875ac06c8ffcd10048a27f77fbd634 Mon Sep 17 00:00:00 2001 From: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:13:22 +0000 Subject: [PATCH] feat: add BigModel provider with native Chat, Responses, and Messages APIs Co-authored-by: Bytechoreographer --- .github/workflows/test-unit.yml | 1 + examples/bigmodel.md | 61 ++ litellm/__init__.py | 5 + litellm/constants.py | 1 + .../get_llm_provider_logic.py | 7 + litellm/llms/bigmodel/__init__.py | 0 litellm/llms/bigmodel/chat/__init__.py | 0 litellm/llms/bigmodel/chat/transformation.py | 55 ++ litellm/llms/bigmodel/common_utils.py | 38 + litellm/llms/bigmodel/messages/__init__.py | 0 .../llms/bigmodel/messages/transformation.py | 149 ++++ litellm/llms/bigmodel/responses/__init__.py | 0 .../llms/bigmodel/responses/transformation.py | 204 +++++ litellm/main.py | 4 + ...odel_prices_and_context_window_backup.json | 76 ++ .../provider_create_fields.json | 28 + litellm/types/utils.py | 1 + litellm/utils.py | 15 + model_prices_and_context_window.json | 76 ++ provider_endpoints_support.json | 17 + .../llms/bigmodel/test_bigmodel_provider.py | 725 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 20 + .../public/assets/logos/bigmodel.png | Bin 0 -> 11339 bytes .../CredentialModal.integration.test.tsx | 42 + .../components/provider_info_helpers.test.tsx | 12 + .../src/components/provider_info_helpers.tsx | 5 + 26 files changed, 1542 insertions(+) create mode 100644 examples/bigmodel.md create mode 100644 litellm/llms/bigmodel/__init__.py create mode 100644 litellm/llms/bigmodel/chat/__init__.py create mode 100644 litellm/llms/bigmodel/chat/transformation.py create mode 100644 litellm/llms/bigmodel/common_utils.py create mode 100644 litellm/llms/bigmodel/messages/__init__.py create mode 100644 litellm/llms/bigmodel/messages/transformation.py create mode 100644 litellm/llms/bigmodel/responses/__init__.py create mode 100644 litellm/llms/bigmodel/responses/transformation.py create mode 100644 tests/unit/llms/bigmodel/test_bigmodel_provider.py create mode 100644 ui/litellm-dashboard/public/assets/logos/bigmodel.png diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 76af64c8ccd..7a10f07899c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -235,6 +235,7 @@ jobs: tests/unit/llms/azure_ai tests/unit/llms/base_llm tests/unit/llms/baseten + tests/unit/llms/bigmodel tests/unit/llms/black_forest_labs tests/unit/llms/brave tests/unit/llms/bytez diff --git a/examples/bigmodel.md b/examples/bigmodel.md new file mode 100644 index 00000000000..1ae64bcbfc4 --- /dev/null +++ b/examples/bigmodel.md @@ -0,0 +1,61 @@ +# BigModel (Zhipu AI) + +Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL + +```yaml +model_list: + - model_name: glm + litellm_params: + model: bigmodel/glm-5.3 + api_key: os.environ/BIGMODEL_API_KEY + api_base: https://open.bigmodel.cn +``` + +One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API + +| Proxy endpoint | BigModel endpoint | +| --- | --- | +| `/v1/chat/completions` | `/api/paas/v4/chat/completions` | +| `/v1/responses` | `/api/v1/responses` | +| `/v1/messages` | `/api/anthropic/v1/messages` | + +`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix + +The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` + +```python +import litellm + +chat = litellm.completion( + model="bigmodel/glm-5.3", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "enabled"}, + reasoning_effort="low", +) + +response = litellm.responses( + model="bigmodel/glm-5.3", + input="Hello", + reasoning={"effort": "low"}, + store=True, +) + +async def messages(): + return await litellm.anthropic.messages.acreate( + model="bigmodel/glm-5.3", + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "adaptive"}, + output_config={"effort": "low"}, + ) +``` + +Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route + +GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters + +Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search + +The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices + +API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/litellm/__init__.py b/litellm/__init__.py index 9ec11dd1291..4221feb11ef 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -634,6 +634,7 @@ watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() +bigmodel_models: Final[set[str]] = set() # mutable-ok: provider registry is populated and extended at runtime deepseek_models: Set = set() tencent_models: Set = set() runwayml_models: Set = set() @@ -873,6 +874,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje xai_models.add(key) elif value.get("litellm_provider") == "zai": zai_models.add(key) + elif value.get("litellm_provider") == "bigmodel": + bigmodel_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": @@ -1097,6 +1100,7 @@ model_list = list( | text_completion_inception_models | xai_models | zai_models + | bigmodel_models | fal_ai_models | deepseek_models | modelscope_models @@ -1208,6 +1212,7 @@ def _build_models_by_provider() -> dict: "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, + "bigmodel": bigmodel_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "tencent": tencent_models, diff --git a/litellm/constants.py b/litellm/constants.py index 2901b7d8948..54ec501471f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1083,6 +1083,7 @@ openai_compatible_providers: Final[list[str]] = [ "xinference", "xai", "zai", + "bigmodel", "together_ai", "fireworks_ai", "empower", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 19c1d5c414c..2af44ad0eab 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -795,6 +795,13 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "bigmodel": + from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url + + ( + api_base, # rebind-ok: existing provider-resolution pipeline resolves the API base in place + dynamic_api_key, # rebind-ok: existing provider-resolution pipeline resolves the API key in place + ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) elif custom_llm_provider == "zai": ( api_base, diff --git a/litellm/llms/bigmodel/__init__.py b/litellm/llms/bigmodel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/__init__.py b/litellm/llms/bigmodel/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py new file mode 100644 index 00000000000..1f721d84dd6 --- /dev/null +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -0,0 +1,55 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.zai.chat.transformation import ZAIChatConfig + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url + +_SUPPORTED_PARAMS: Final = ( + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "temperature", + "top_p", + "stop", + "tools", + "tool_choice", + "response_format", + "thinking", + "reasoning_effort", +) + + +class BigModelChatConfig(ZAIChatConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: + return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + mapped: Final = { + **optional_params, + **{ + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in _SUPPORTED_PARAMS and key != "thinking" + }, + } + if "thinking" not in non_default_params: + return mapped + return { + **mapped, + "extra_body": {"thinking": non_default_params["thinking"]}, + } diff --git a/litellm/llms/bigmodel/common_utils.py b/litellm/llms/bigmodel/common_utils.py new file mode 100644 index 00000000000..d17f478ed29 --- /dev/null +++ b/litellm/llms/bigmodel/common_utils.py @@ -0,0 +1,38 @@ +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit, urlunsplit + +from litellm.secret_managers.main import get_secret_str + +BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn" +_API_PATHS: Final = ( + "/api/paas/v4/chat/completions", + "/api/anthropic/v1/messages", + "/api/anthropic/v1", + "/api/v1/responses", + "/api/paas/v4", + "/api/anthropic", + "/api/v1", + "/api", +) +_ENDPOINT_PATHS: Final = MappingProxyType( + { + "chat": "/api/paas/v4", + "responses": "/api/v1/responses", + "messages": "/api/anthropic/v1/messages", + } +) + + +def get_bigmodel_api_key(api_key: str | None) -> str | None: + return api_key or get_secret_str("BIGMODEL_API_KEY") + + +def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str: + resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE + parsed: Final = urlsplit(resolved_base) + path: Final = parsed.path.rstrip("/") + root: Final = next((path.removesuffix(suffix) for suffix in _API_PATHS if path.endswith(suffix)), path) + return urlunsplit( + (parsed.scheme, parsed.netloc, f"{root}{_ENDPOINT_PATHS[endpoint]}", parsed.query, parsed.fragment) + ) diff --git a/litellm/llms/bigmodel/messages/__init__.py b/litellm/llms/bigmodel/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/messages/transformation.py b/litellm/llms/bigmodel/messages/transformation.py new file mode 100644 index 00000000000..2cac2d3b899 --- /dev/null +++ b/litellm/llms/bigmodel/messages/transformation.py @@ -0,0 +1,149 @@ +import json +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Final + +import httpx +from openai._streaming import ServerSentEvent, SSEDecoder +from pydantic import JsonValue, TypeAdapter + +from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai_like.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, + OpenAILikeAnthropicMessagesConfig, +) +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: # mutable-ok: native JSON response + usage: Final = body.get("usage") + if not isinstance(usage, dict): + return dict(body) + server_tools: Final = usage.get("server_tool_use") + if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools: + return dict(body) + normalized: Final[JsonValue] = { + **body, + "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}}, + } + return normalized + + +class _BigModelMessagesStream(httpx.AsyncByteStream): + def __init__(self, response: httpx.Response) -> None: + self._response = response + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()): + yield _normalize_event(event) + + async def aclose(self) -> None: + await self._response.aclose() + + +def _normalize_event(event: ServerSentEvent) -> bytes: + body: Final = _JSON_OBJECT.validate_json(event.data) + message: Final = body.get("message") + normalized: Final[JsonValue] = ( + {**body, "message": _normalize_message(message)} + if body.get("type") == "message_start" and isinstance(message, dict) + else _normalize_message(body) + ) + metadata: Final = "".join( + f"{name}: {value}\n" + for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry)) + if value is not None + ) + return f"{metadata}data: {json.dumps(normalized)}\n\n".encode() + + +class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig): + @property + def custom_llm_provider(self) -> str: + return "bigmodel" + + def is_native_messages_route(self) -> bool: + return True + + def should_strip_billing_metadata(self) -> bool: + return True + + def transform_anthropic_messages_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> AnthropicMessagesResponse: + normalized: Final = httpx.Response( + raw_response.status_code, + json=_normalize_message(_JSON_OBJECT.validate_json(raw_response.content)), + request=raw_response.request, + ) + return super().transform_anthropic_messages_response(model, normalized, logging_obj) + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict[str, object], # mutable-ok: inherited native stream API takes a request dict + litellm_logging_obj: Logging, + ) -> AsyncIterator[bytes]: + normalized: Final = httpx.Response( + httpx_response.status_code, + headers={ + key: value + for key, value in httpx_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + stream=_BigModelMessagesStream(httpx_response), + request=httpx_response.request, + ) + return super().get_async_streaming_response_iterator( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # native parent yields bytes but its iterator annotation omits the element type + model, normalized, request_body, litellm_logging_obj + ) + + @staticmethod + def _translate_reasoning_effort_to_anthropic( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + @staticmethod + def _translate_adaptive_effort_for_non_adaptive_model( + model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str + ) -> None: + return None + + def validate_anthropic_messages_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(api_key) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")}, + "x-api-key": resolved_key, + "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, + "content-type": "application/json", + }, api_base + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + stream: bool | None = None, + ) -> str: + return get_bigmodel_url(api_base, "messages") diff --git a/litellm/llms/bigmodel/responses/__init__.py b/litellm/llms/bigmodel/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/bigmodel/responses/transformation.py b/litellm/llms/bigmodel/responses/transformation.py new file mode 100644 index 00000000000..a61f006371e --- /dev/null +++ b/litellm/llms/bigmodel/responses/transformation.py @@ -0,0 +1,204 @@ +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import JsonValue, TypeAdapter +from typing_extensions import Never + +from litellm.exceptions import AuthenticationError, UnsupportedParamsError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +from ..common_utils import get_bigmodel_api_key, get_bigmodel_url + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_SUPPORTED_PARAMS: Final = ( + "input", + "model", + "instructions", + "stream", + "temperature", + "top_p", + "max_output_tokens", + "tools", + "tool_choice", + "reasoning", + "text", + "prompt_cache_key", + "previous_response_id", + "store", + "extra_headers", + "extra_query", + "extra_body", + "timeout", +) + + +def _normalize_output_item(item: JsonValue) -> JsonValue: + if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"): + return item + content: Final = item.get("content") + normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item + if not isinstance(content, dict): + return normalized + return {**normalized, "content": [content]} + + +def _normalize_response( + body: Mapping[str, JsonValue], +) -> dict[str, JsonValue]: # mutable-ok: JSON serialization boundary + usage: Final = body.get("usage") + output: Final = body.get("output") + tools: Final = body.get("tools") + tool_choice: Final = body.get("tool_choice") + input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None + output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None + normalized_usage: Final = ( + { + "total_tokens": input_tokens + output_tokens, + **usage, + } + if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int) + else usage + ) + normalized: Final[JsonValue] = { + "parallel_tool_calls": False, + **body, + "tools": [] if tools is None else tools, + "tool_choice": "auto" if tool_choice is None else tool_choice, + **({"usage": normalized_usage} if "usage" in body else {}), + **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}), + } + return normalized + + +def _normalize_stream_part(part: JsonValue) -> JsonValue: + if not isinstance(part, dict): + return part + if part.get("type") == "output_text": + return { + "annotations": [], + "logprobs": None, + **part, + } + if part.get("type") == "reasoning_text": + return {"reasoning": part.get("text", ""), **part} + return part + + +class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BIGMODEL + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return False + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: inherited API requires a list + return list(_SUPPORTED_PARAMS) + + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS} + + def validate_environment( + self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None + ) -> dict[str, str]: # mutable-ok: HTTP handler merges forwarded headers + resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None) + if resolved_key is None: + raise AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) + return { + **headers, + "Authorization": f"Bearer {resolved_key}", + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + return get_bigmodel_url(api_base, "responses") + + def transform_response_api_response( + self, model: str, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + normalized: Final = _normalize_response(_JSON_OBJECT.validate_json(raw_response.content)) + normalized_response: Final = httpx.Response( + status_code=raw_response.status_code, + headers={ + key: value + for key, value in raw_response.headers.items() + if key.lower() not in ("content-encoding", "content-length") + }, + json=normalized, + request=raw_response.request, + ) + return super().transform_response_api_response(model, normalized_response, logging_obj) + + def transform_get_response_api_response( + self, raw_response: httpx.Response, logging_obj: Logging + ) -> ResponsesAPIResponse: + return self.transform_response_api_response("", raw_response, logging_obj) + + def transform_streaming_response( + self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging + ) -> ResponsesAPIStreamingResponse: + chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk) + response: Final = chunk.get("response") + item: Final = chunk.get("item") + normalized: Final[JsonValue] = { + **chunk, + **({"response": _normalize_response(response)} if isinstance(response, dict) else {}), + **({"item": _normalize_output_item(item)} if "item" in chunk else {}), + } + event_model: Final = self.get_event_model_class(str(normalized.get("type"))) + match chunk.get("type"): + case "response.output_text.done": + return event_model.model_validate({"logprobs": [], **normalized}) + case "response.content_part.done": + return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))}) + case "error": + error: Final = chunk.get("error") + error_fields: Final = error if isinstance(error, dict) else chunk + normalized_error: Final[JsonValue] = { + **normalized, + "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"}, + } + return event_model.model_validate(normalized_error) + case _: + return event_model.model_validate(normalized) + + def transform_cancel_response_api_request( + self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str] + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400) + + def transform_compact_response_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: Mapping[str, object], + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> Never: + raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400) diff --git a/litellm/main.py b/litellm/main.py index 23eac8ce0aa..2f4ed844965 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2666,6 +2666,10 @@ def _complete_custom_openai( # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there api_base = _resolve_openai_api_base(api_base) + if custom_llm_provider == "bigmodel" and not api_key: + raise litellm.AuthenticationError( + message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model + ) organization = ( organization or litellm.organization diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b820b058eb3..38c86d266fc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76005,5 +76005,81 @@ "high", "max" ] + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d96f2976508..885220ac959 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -4144,5 +4144,33 @@ } ], "default_model_placeholder": "cursor/claude-4-sonnet" + }, + { + "provider": "BIGMODEL", + "provider_display_name": "BigModel (Zhipu AI)", + "litellm_provider": "bigmodel", + "credential_fields": [ + { + "key": "api_key", + "label": "BigModel (Zhipu AI) API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://open.bigmodel.cn", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": "https://open.bigmodel.cn" + } + ], + "default_model_placeholder": "bigmodel/glm-5.3" } ] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7e8645fd0dd..374aad26bd0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4422,6 +4422,7 @@ class LlmProviders(str, Enum): JINA_AI = "jina_ai" XAI = "xai" ZAI = "zai" + BIGMODEL = "bigmodel" CUSTOM_OPENAI = "custom_openai" TEXT_COMPLETION_OPENAI = "text-completion-openai" COHERE = "cohere" diff --git a/litellm/utils.py b/litellm/utils.py index 781aa5f4fce..35ab33750e8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9133,6 +9133,7 @@ class ProviderConfigManager: LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), LlmProviders.XAI: (lambda: litellm.XAIChatConfig(), False), LlmProviders.ZAI: (lambda: litellm.ZAIChatConfig(), False), + LlmProviders.BIGMODEL: (ProviderConfigManager._get_bigmodel_chat_config, False), LlmProviders.OPENCODE_GO: (ProviderConfigManager.get_opencode_go_config, True), LlmProviders.LAMBDA_AI: (lambda: litellm.LambdaAIChatConfig(), False), LlmProviders.INCEPTION: (lambda: litellm.InceptionChatConfig(), False), @@ -9325,6 +9326,12 @@ class ProviderConfigManager: return get_bedrock_chat_config(model=model) + @staticmethod + def _get_bigmodel_chat_config() -> BaseConfig: + from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig + + return BigModelChatConfig() + @staticmethod def get_opencode_go_config(model: str) -> BaseConfig: from litellm.llms.opencode.chat.transformation import go_chat_config @@ -9645,6 +9652,10 @@ class ProviderConfigManager: model: str, provider: LlmProviders, ) -> BaseAnthropicMessagesConfig | None: + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig + + return BigModelMessagesConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.messages.transformation import OpenCodeGoMessagesConfig @@ -9876,6 +9887,10 @@ class ProviderConfigManager: if provider is None: return None + if provider == LlmProviders.BIGMODEL: + from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig + + return BigModelResponsesAPIConfig() if provider == LlmProviders.OPENCODE_GO: from litellm.llms.opencode.common_utils import go_endpoint from litellm.llms.opencode.responses.transformation import OpenCodeGoResponsesConfig diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b820b058eb3..38c86d266fc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76005,5 +76005,81 @@ "high", "max" ] + }, + "bigmodel/glm-5.3": { + "input_cost_per_token": 1.1428571428571428e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2.857142857142857e-07, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flash": { + "input_cost_per_token": 1.142857142857143e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3.2857142857142855e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true + }, + "bigmodel/glm-5.3-flashx": { + "input_cost_per_token": 2.857142857142857e-07, + "output_cost_per_token": 1e-06, + "cache_read_input_token_cost": 8.142857142857142e-08, + "litellm_provider": "bigmodel", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "source": "https://docs.bigmodel.cn/cn/guide/start/pricing", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "thinking_always_on": true, + "supports_vision": true, + "supports_video_input": true, + "metadata": { + "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded." + }, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages" + ], + "supports_native_streaming": true } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e62b74aa87d..b02568fcda9 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3289,6 +3289,23 @@ "rerank": false, "a2a": false } + }, + "bigmodel": { + "display_name": "BigModel (Zhipu AI) (`bigmodel`)", + "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } } }, "endpoints": { diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py new file mode 100644 index 00000000000..f53673830b2 --- /dev/null +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -0,0 +1,725 @@ +import gzip +import json +from collections.abc import AsyncIterator, Iterator, Mapping +from functools import partial +from itertools import chain +from typing import Final, Literal + +import httpx +import pytest +import respx +from openai.types.responses import Response as OpenAIResponse +from pydantic import JsonValue + +import litellm +from litellm.llms.bigmodel.chat.transformation import BigModelChatConfig +from litellm.llms.bigmodel.common_utils import get_bigmodel_url +from litellm.llms.bigmodel.messages.transformation import BigModelMessagesConfig +from litellm.llms.bigmodel.responses.transformation import BigModelResponsesAPIConfig +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +MODEL: Final = "glm-5.3" +BASE: Final = "https://open.bigmodel.cn" +KEY: Final = "test-bigmodel-key" +BASE_PATHS: Final = ( + "", + "/", + "/api", + "/api/paas/v4", + "/api/paas/v4/", + "/api/paas/v4/chat/completions", + "/api/v1", + "/api/v1/responses", + "/api/anthropic", + "/api/anthropic/v1", + "/api/anthropic/v1/messages", +) + + +class _ByteChunks(httpx.AsyncByteStream): + def __init__(self, chunks: tuple[bytes, ...]) -> None: + self._chunks = chunks + self.is_closed = False + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._chunks: + yield chunk + + async def aclose(self) -> None: + self.is_closed = True + + +@pytest.fixture(autouse=True) +def bigmodel_environment(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("BIGMODEL_API_KEY", KEY) + monkeypatch.delenv("BIGMODEL_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + yield + + +def response_payload() -> Mapping[str, JsonValue]: + return { + "id": "resp_bigmodel_test", + "created_at": 1, + "object": "response", + "model": MODEL, + "status": "completed", + "parallel_tool_calls": False, + "tools": None, + "tool_choice": None, + "output": [ + {"id": "rs_test", "type": "reasoning", "content": {"type": "reasoning_text", "text": "Thinking"}}, + { + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": {"type": "output_text", "text": "Hello", "annotations": []}, + }, + ], + "usage": { + "input_tokens": 20, + "output_tokens": 10, + "input_tokens_details": {"cached_tokens": 5}, + "output_tokens_details": {"reasoning_tokens": 3}, + }, + } + + +def test_provider_resolution_and_all_native_endpoints() -> None: + model, provider, key, base = litellm.get_llm_provider(f"bigmodel/{MODEL}") + assert (model, provider, key, base) == (MODEL, "bigmodel", KEY, f"{BASE}/api/paas/v4") + assert isinstance(ProviderConfigManager.get_provider_chat_config(MODEL, LlmProviders.BIGMODEL), BigModelChatConfig) + assert isinstance( + ProviderConfigManager.get_provider_responses_api_config("bigmodel", MODEL), BigModelResponsesAPIConfig + ) + assert isinstance( + ProviderConfigManager.get_provider_anthropic_messages_config(MODEL, LlmProviders.BIGMODEL), + BigModelMessagesConfig, + ) + assert "bigmodel" in litellm.openai_compatible_providers + assert "bigmodel" in litellm.provider_list + assert litellm.get_llm_provider(f"zai/{MODEL}")[3] == "https://api.z.ai/api/paas/v4" + + +@pytest.mark.parametrize("path", BASE_PATHS) +def test_one_base_selects_each_api_without_duplicate_segments(path: str) -> None: + assert get_bigmodel_url(f"{BASE}{path}", "chat") == f"{BASE}/api/paas/v4" + assert get_bigmodel_url(f"{BASE}{path}", "responses") == f"{BASE}/api/v1/responses" + assert get_bigmodel_url(f"{BASE}{path}", "messages") == f"{BASE}/api/anthropic/v1/messages" + + +def test_custom_base_and_environment_precedence(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BIGMODEL_API_BASE", "https://env.example/prefix/api/v1") + assert get_bigmodel_url(None, "messages") == "https://env.example/prefix/api/anthropic/v1/messages" + assert ( + get_bigmodel_url("https://override.example/prefix/api/anthropic/", "chat") + == "https://override.example/prefix/api/paas/v4" + ) + assert ( + BigModelResponsesAPIConfig().validate_environment({}, MODEL, GenericLiteLLMParams(api_key="explicit"))[ + "Authorization" + ] + == "Bearer explicit" + ) + assert BigModelResponsesAPIConfig().validate_environment({}, MODEL, None)["Authorization"] == f"Bearer {KEY}" + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_chat_uses_native_endpoint_and_maps_token_limit(is_async: bool) -> None: + route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "object": "chat.completion", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + } + ) + params: Final = { + "model": f"bigmodel/{MODEL}", + "messages": [{"role": "user", "content": "Hello"}], + "api_base": f"{BASE}/api/anthropic", + "max_completion_tokens": 10, + "thinking": {"type": "enabled"}, + "reasoning_effort": "low", + "extra_body": {"clear_thinking": False}, + } + response: Final = await litellm.acompletion(**params) if is_async else litellm.completion(**params) + assert response.choices[0].message.content == "Hello" + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert body["model"] == MODEL + assert body["max_tokens"] == 10 + assert "max_completion_tokens" not in body + assert body["thinking"] == {"type": "enabled"} + assert body["reasoning_effort"] == "low" + assert body["clear_thinking"] is False + assert request.headers["authorization"] == f"Bearer {KEY}" + assert route.call_count == 1 + if is_async: + health: Final = await litellm.ahealth_check(params, mode="chat") + assert "error" not in health + assert json.loads(route.calls.last.request.content)["thinking"] == {"type": "enabled"} + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_responses_native_body_text_usage_and_continuation(is_async: bool) -> None: + route: Final = respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + params: Final = { + "model": f"bigmodel/{MODEL}", + "input": "Hello", + "api_base": BASE, + "max_output_tokens": 1, + "reasoning": {"effort": "max"}, + } + response: Final = await litellm.aresponses(**params) if is_async else litellm.responses(**params) + assert isinstance(response, ResponsesAPIResponse) + assert response.output_text == "Hello" + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.total_tokens == 30 + assert response.usage.input_tokens_details.cached_tokens == 5 + assert response.usage.output_tokens_details.reasoning_tokens == 3 + assert response.tools == [] + assert response.tool_choice == "auto" + OpenAIResponse.model_validate(response.model_dump()) + first_body: Final = json.loads(route.calls.last.request.content) + assert first_body["model"] == MODEL + assert first_body["max_output_tokens"] == 1 + assert first_body["reasoning"] == {"effort": "max"} + assert first_body.get("store") is not True + assert "messages" not in first_body + assert route.calls.last.request.headers["authorization"] == f"Bearer {KEY}" + followup: Final = {**params, "previous_response_id": response.id, "store": True} + if is_async: + await litellm.aresponses(**followup) + else: + litellm.responses(**followup) + assert json.loads(route.calls.last.request.content)["previous_response_id"] == "resp_bigmodel_test" + assert json.loads(route.calls.last.request.content)["store"] is True + + +@pytest.mark.asyncio +@respx.mock +async def test_responses_decodes_compressed_response_once() -> None: + respx.post(f"{BASE}/api/v1/responses").respond( + content=gzip.compress(json.dumps(response_payload()).encode()), + headers={"content-encoding": "gzip", "content-type": "application/json"}, + ) + response: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello") + assert response.output_text == "Hello" + assert response.usage.total_tokens == 30 + + +@pytest.mark.parametrize( + "reasoning_params", + ( + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}, + {"thinking": {"type": "enabled", "budget_tokens": 1024}}, + {"reasoning_effort": "low"}, + ), +) +def test_messages_preserves_native_reasoning_params(reasoning_params: Mapping[str, JsonValue]) -> None: + payload: Final = BigModelMessagesConfig().transform_anthropic_messages_request( + model=MODEL, + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 2048, **reasoning_params}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert {key: payload.get(key) for key in reasoning_params} == reasoning_params + + +@pytest.mark.asyncio +@respx.mock +@pytest.mark.parametrize("web_fetch_requests", (None, 2)) +async def test_messages_forward_native_tools_thinking_and_usage(web_fetch_requests: int | None) -> None: + route: Final = respx.post(f"{BASE}/api/anthropic/v1/messages").respond( + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 20, + "output_tokens": 10, + "server_tool_use": { + "web_search_requests": 1, + **({"web_fetch_requests": web_fetch_requests} if web_fetch_requests is not None else {}), + }, + }, + } + ) + tools: Final = [{"name": "weather", "input_schema": {"type": "object", "properties": {}}}] + params: Final = { + "model": f"bigmodel/{MODEL}", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hello"}], + "tools": tools, + "thinking": {"type": "enabled", "budget_tokens": 50}, + "api_base": f"{BASE}/api/paas/v4", + "provider_specific_header": { + "custom_llm_provider": "bigmodel", + "native_anthropic_messages_only": True, + "extra_headers": {"x-native-message": "preserved"}, + }, + } + response: Final = await litellm.anthropic.messages.acreate(**params) + assert response["content"][0]["text"] == "Hello" + assert response["usage"]["server_tool_use"] == { + "web_search_requests": 1, + "web_fetch_requests": web_fetch_requests or 0, + } + body: Final = json.loads(route.calls.last.request.content) + assert body["model"] == MODEL + assert body["tools"] == tools + assert body["thinking"]["type"] == "enabled" + assert "input" not in body + assert route.calls.last.request.headers["x-api-key"] == KEY + assert route.calls.last.request.headers["x-native-message"] == "preserved" + + +@pytest.mark.parametrize( + "terminal_type,status", + (("response.completed", "completed"), ("response.incomplete", "incomplete")), +) +@pytest.mark.parametrize("include_logprobs", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_responses_sse_without_done_preserves_terminal_usage( + terminal_type: str, status: str, include_logprobs: bool +) -> None: + logprobs: Final = [{"token": "Hello", "bytes": [72, 101, 108, 108, 111], "logprob": -0.1, "top_logprobs": []}] + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": { + **{key: value for key, value in response_payload().items() if key != "parallel_tool_calls"}, + "status": "in_progress", + "output": [], + "usage": None, + }, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "delta": "Hello", + }, + { + "type": "response.content_part.done", + "sequence_number": 2, + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "Hello"}, + }, + { + "type": "response.content_part.done", + "sequence_number": 3, + "item_id": "rs_test", + "output_index": 1, + "content_index": 0, + "part": {"type": "reasoning_text", "text": "Thinking"}, + }, + { + "type": "response.output_text.done", + "sequence_number": 4, + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "text": "Hello", + **({"logprobs": logprobs} if include_logprobs else {}), + }, + {"type": terminal_type, "sequence_number": 5, "response": {**response_payload(), "status": status}}, + ) + sse: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + respx.post(f"{BASE}/api/v1/responses").respond(content=sse, headers={"content-type": "text/event-stream"}) + stream: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + chunks: Final = tuple([chunk async for chunk in stream]) + assert chunks[-1].type == terminal_type + assert chunks[-1].response.status == status + assert chunks[-1].response.usage.total_tokens == 30 + assert chunks[-1].response.output_text == "Hello" + OpenAIResponse.model_validate(chunks[-1].response.model_dump()) + assert chunks[0].response.parallel_tool_calls is False + assert chunks[-2].logprobs == (logprobs if include_logprobs else []) + assert tuple(chunk.delta for chunk in chunks if chunk.type == "response.output_text.delta") == ("Hello",) + assert tuple(chunk.part.text for chunk in chunks if chunk.type == "response.content_part.done") == ( + "Hello", + "Thinking", + ) + + +@respx.mock +def test_get_response_uses_bigmodel_url_and_normalizes_usage() -> None: + route: Final = respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + response: Final = litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", store=True) + respx.get(f"{BASE}/api/v1/responses/resp_bigmodel_test").respond(json=response_payload()) + retrieved: Final = litellm.get_responses(response_id=response.id) + assert retrieved.output_text == "Hello" + assert retrieved.usage.total_tokens == 30 + assert route.call_count == 1 + + +@pytest.mark.parametrize("provider_api", ("chat", "responses", "messages")) +def test_missing_bigmodel_key_does_not_use_other_provider_credentials( + provider_api: Literal["chat", "responses", "messages"], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("BIGMODEL_API_KEY") + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "unrelated-anthropic-key") + call: Final = { + "chat": partial(litellm.completion, model=f"bigmodel/{MODEL}", messages=[{"role": "user", "content": "Hello"}]), + "responses": partial(BigModelResponsesAPIConfig().validate_environment, {}, MODEL, None), + "messages": partial(BigModelMessagesConfig().validate_anthropic_messages_environment, {}, MODEL, (), {}, {}), + }[provider_api] + with pytest.raises(litellm.AuthenticationError, match="BIGMODEL_API_KEY"): + call() + + +@pytest.mark.parametrize("event_type", ("response.failed", "error")) +@pytest.mark.asyncio +@respx.mock +async def test_responses_failure_events_raise_instead_of_completing(event_type: str) -> None: + event: Final = ( + {"type": event_type, "sequence_number": 1, "code": "server_error", "message": "Provider failed"} + if event_type == "error" + else { + "type": event_type, + "sequence_number": 1, + "response": { + **response_payload(), + "status": "failed", + "error": {"code": "server_error", "message": "Provider failed"}, + }, + } + ) + respx.post(f"{BASE}/api/v1/responses").respond( + content=f"event: {event_type}\ndata: {json.dumps(event)}\n\n", + headers={"content-type": "text/event-stream"}, + ) + stream: Final = await litellm.aresponses(model=f"bigmodel/{MODEL}", input="Hello", stream=True) + with pytest.raises(litellm.exceptions.MidStreamFallbackError, match="Provider failed"): + await anext(stream) + + +@respx.mock +def test_responses_preserve_tools_multimodal_input_and_existing_usage_total() -> None: + payload: Final = response_payload() + route: Final = respx.post(f"{BASE}/api/v1/responses").respond( + json={**payload, "usage": {**payload["usage"], "total_tokens": 31}} + ) + input_items: Final = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this"}, + {"type": "input_image", "image_url": "https://example.com/image.png"}, + ], + } + ] + tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object", "properties": {}}}] + response: Final = litellm.responses( + model=f"bigmodel/{MODEL}", + input=input_items, + tools=tools, + tool_choice="auto", + text={"format": {"type": "json_object"}}, + ) + assert response.usage.total_tokens == 31 + assert response.output[0].content[0].text == "Thinking" + body: Final = json.loads(route.calls.last.request.content) + assert body["input"] == input_items + assert body["tools"] == tools + assert body["text"] == {"format": {"type": "json_object"}} + + +@respx.mock +def test_responses_preserve_explicit_tool_configuration() -> None: + tools: Final = [{"type": "function", "name": "weather", "parameters": {"type": "object"}, "strict": False}] + tool_choice: Final = {"type": "function", "name": "weather"} + reasoning: Final = { + "id": "rs_test", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Planning"}], + "content": [{"type": "reasoning_text", "text": "Thinking"}], + } + tool_call: Final = { + "type": "function_call", + "id": "fc_test", + "call_id": "call_test", + "name": "weather", + "arguments": "{}", + "status": "completed", + } + respx.post(f"{BASE}/api/v1/responses").respond( + json={ + **response_payload(), + "output": [reasoning, tool_call], + "tools": tools, + "tool_choice": tool_choice, + "parallel_tool_calls": True, + } + ) + response: Final = litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", tools=tools, tool_choice=tool_choice) + validated: Final = OpenAIResponse.model_validate(response.model_dump()) + assert validated.tools[0].name == "weather" + assert validated.tool_choice.name == "weather" + assert validated.parallel_tool_calls is True + assert validated.output[0].summary[0].text == reasoning["summary"][0]["text"] + assert validated.output[0].content[0].text == reasoning["content"][0]["text"] + assert validated.output[1].model_dump(exclude_none=True) == tool_call + + +@pytest.mark.asyncio +@respx.mock +async def test_router_routes_one_model_configuration_to_three_native_endpoints() -> None: + chat: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "created": 1, + "model": MODEL, + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + } + ) + responses: Final = respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + messages: Final = respx.post(f"{BASE}/api/anthropic/v1/messages").respond( + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 10}, + } + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "glm-native", + "litellm_params": {"model": f"bigmodel/{MODEL}", "api_key": KEY, "api_base": BASE}, + } + ], + num_retries=0, + ) + chat_result: Final = await router.acompletion(model="glm-native", messages=[{"role": "user", "content": "Hello"}]) + responses_result: Final = await router.aresponses(model="glm-native", input="Hello") + messages_result: Final = await router.aanthropic_messages( + model="glm-native", max_tokens=10, messages=[{"role": "user", "content": "Hello"}] + ) + assert ( + chat_result.choices[0].message.content + == responses_result.output_text + == messages_result["content"][0]["text"] + == "Hello" + ) + assert tuple(route.call_count for route in (chat, responses, messages)) == (1, 1, 1) + assert all(json.loads(route.calls.last.request.content)["model"] == MODEL for route in (chat, responses, messages)) + assert ( + chat.calls.last.request.headers["authorization"] + == responses.calls.last.request.headers["authorization"] + == f"Bearer {KEY}" + ) + assert messages.calls.last.request.headers["x-api-key"] == KEY + + +@pytest.mark.asyncio +@respx.mock +async def test_chat_stream_preserves_deltas_and_final_usage() -> None: + chunks: Final = ( + { + "id": "chat_test", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": {"reasoning_content": "Thinking"}, "finish_reason": None}], + }, + { + "id": "chat_test", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": None}], + }, + { + "id": "chat_test", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + ) + respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + content="".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n", + headers={"content-type": "text/event-stream"}, + ) + stream: Final = await litellm.acompletion( + model=f"bigmodel/{MODEL}", + messages=[{"role": "user", "content": "Hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + output: Final = tuple([chunk async for chunk in stream]) + assert ( + "".join(choice.delta.content or "" for choice in chain.from_iterable(chunk.choices for chunk in output)) + == "Hello" + ) + assert ( + "".join( + getattr(choice.delta, "reasoning_content", "") or "" + for choice in chain.from_iterable(chunk.choices for chunk in output) + ) + == "Thinking" + ) + assert tuple(chunk.usage.total_tokens for chunk in output if hasattr(chunk, "usage") and chunk.usage) == (30,) + + +@pytest.mark.asyncio +@respx.mock +async def test_messages_stream_preserves_native_events() -> None: + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": MODEL, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 20, + "output_tokens": 0, + "server_tool_use": {"web_search_requests": 0}, + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "先思考"}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "test-signature"}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "text_delta", "text": "你好 🌏"}}, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 10}, + }, + {"type": "message_stop"}, + ) + content: Final = ( + "id: bigmodel-event\nretry: 1000\n" + + "".join(f"event: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n" for event in events) + ).encode() + upstream: Final = _ByteChunks(tuple(content[index : index + 7] for index in range(0, len(content), 7))) + respx.post(f"{BASE}/api/anthropic/v1/messages").mock( + return_value=httpx.Response( + 200, + stream=upstream, + headers={"content-type": "text/event-stream"}, + ) + ) + stream: Final = await litellm.anthropic.messages.acreate( + model=f"bigmodel/{MODEL}", + max_tokens=100, + messages=[{"role": "user", "content": "Hello"}], + stream=True, + ) + output: Final = tuple([chunk async for chunk in stream]) + parsed: Final = tuple( + json.loads(line.removeprefix("data: ")) + for line in chain.from_iterable(chunk.decode().splitlines() for chunk in output) + if line.startswith("data: ") + ) + assert tuple(chunk["type"] for chunk in parsed) == tuple(event["type"] for event in events) + assert tuple(event["delta"] for event in parsed if event["type"] == "content_block_delta") == ( + {"type": "thinking_delta", "thinking": "先思考"}, + {"type": "signature_delta", "signature": "test-signature"}, + {"type": "text_delta", "text": "你好 🌏"}, + ) + assert parsed[-2]["usage"]["output_tokens"] == 10 + assert parsed[0]["message"]["usage"]["server_tool_use"] == { + "web_search_requests": 0, + "web_fetch_requests": 0, + } + assert b"id: bigmodel-event\n" in output[0] + assert b"retry: 1000\n" in output[0] + assert upstream.is_closed + + +@respx.mock +def test_stored_response_lifecycle_reuses_base_and_unwraps_id() -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json=response_payload()) + response: Final = litellm.responses(model=f"bigmodel/{MODEL}", input="Hello", store=True) + listing: Final = respx.get(f"{BASE}/api/v1/responses/resp_bigmodel_test/input_items").respond( + json={"object": "list", "data": [], "has_more": False} + ) + deletion: Final = respx.delete(f"{BASE}/api/v1/responses/resp_bigmodel_test").respond( + json={"id": "resp_bigmodel_test", "object": "response.deleted", "deleted": True} + ) + assert litellm.list_input_items(response_id=response.id)["data"] == [] + assert litellm.delete_responses(response_id=response.id).deleted is True + assert listing.call_count == deletion.call_count == 1 + assert deletion.calls.last.request.headers["authorization"] == f"Bearer {KEY}" + + +@pytest.mark.parametrize("operation", ("cancel", "compact")) +@respx.mock +def test_unsupported_responses_operations_fail_without_an_upstream_request(operation: str) -> None: + call: Final = ( + partial(litellm.cancel_responses, response_id="resp_test", custom_llm_provider="bigmodel") + if operation == "cancel" + else partial(litellm.compact_responses, model=f"bigmodel/{MODEL}", input="Hello") + ) + with pytest.raises(litellm.UnsupportedParamsError, match="BigModel"): + call() + assert respx.calls.call_count == 0 + + +@pytest.mark.parametrize("model", ("glm-5.3", "glm-5.3-flash", "glm-5.3-flashx")) +def test_bigmodel_model_metadata_and_cached_token_costs(model: str) -> None: + entry: Final = litellm.model_cost[f"bigmodel/{model}"] + info: Final = litellm.get_model_info(model=model, custom_llm_provider="bigmodel") + assert info["litellm_provider"] == "bigmodel" + assert info["max_input_tokens"] == entry["max_input_tokens"] + assert info["max_output_tokens"] == entry["max_output_tokens"] + assert info["supports_reasoning"] + assert info["supports_function_calling"] + assert bool(info.get("supports_vision")) == bool(entry.get("supports_vision")) + prompt_cost, output_cost = litellm.cost_per_token( + model=model, + custom_llm_provider="bigmodel", + prompt_tokens=100, + completion_tokens=20, + cache_read_input_tokens=40, + ) + assert prompt_cost == pytest.approx(60 * entry["input_cost_per_token"] + 40 * entry["cache_read_input_token_cost"]) + assert output_cost == pytest.approx(20 * entry["output_cost_per_token"]) + assert prompt_cost > 0 + assert output_cost > 0 diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index cc61b6f2fcf..f99c9da5d05 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -25,6 +25,26 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreate from litellm.types.utils import LlmProviders +def test_bigmodel_fields_require_private_key_and_share_default_base_with_adapter() -> None: + from litellm.llms.bigmodel.common_utils import BIGMODEL_API_BASE + + app: Final = FastAPI() + app.include_router(router) + response: Final = TestClient(app).get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = TypeAdapter(list[ProviderCreateInfo]).validate_python(response.json()) + provider: Final = next(item for item in providers if item.litellm_provider == "bigmodel") + fields: Final = {field.key: field for field in provider.credential_fields} + assert provider.provider_display_name == "BigModel (Zhipu AI)" + assert fields["api_key"].required is True + assert fields["api_key"].field_type == "password" + assert fields["api_base"].required is False + assert fields["api_base"].default_value == BIGMODEL_API_BASE + assert fields["api_base"].placeholder == BIGMODEL_API_BASE + assert provider.default_model_placeholder is not None + assert provider.default_model_placeholder.startswith("bigmodel/") + + def test_fuse_presets_route_serves_the_shared_catalog_without_authentication() -> None: app: Final = FastAPI() app.include_router(router) diff --git a/ui/litellm-dashboard/public/assets/logos/bigmodel.png b/ui/litellm-dashboard/public/assets/logos/bigmodel.png new file mode 100644 index 0000000000000000000000000000000000000000..f22030f85b8fe85456e8abd027952c44edf16227 GIT binary patch literal 11339 zcmWk!2|QHa7k{&07|S60zEjyLgc)nL5VA#xA}Pwg#MqaFC_7`xmO_by7?dAT_9cn2 zZ`rpoX8F(m``)|n^Ld~1?mhS1dzO3dO|~?@&cwjW0001!v5~$N^^N`i1A|b@d*DJ< z0Du6NW;O=b{mir+hPZ(o3WahfiK1lt+vCON)jP{d*24+!Ns5ktNMwrlt3B<2H5I4X z(=#%aj7XyRX6-57{=2?SbbaxUa86OUJ%4mc^2%AcQ?$FiPZG14yYhG$w?^DQCOL;~ z%Q?>J-dl7C+;=Fjnd*jyQt0U z)iZ5R;?`W(AYS+0ica>+s5eJm!TZSjD*>^TjsX^gdNYwGOsvRqkzLZ$QxKF}*(le#r`=k2z6 z zT^BzO-~+Nx65`Li(s!xf-(ydwapyO}HWS{SIVSGt6tC9(I@2jyjjAGeXB^&7J8+2K z`Z9iYr(& z?^hn!#cn_Ov~&B-Zd}}1Ml;c?@-V$;KR4%WWa&Jt;H+W%lsds|%6F_D{<~53&*{xx zNaU$c$l;UlV~@&%>-RSvRvp_t+BWmryzy+yF=5xBbVJvDP0MZ7CT3f~aq+hAPGmIR zFm%)9$*#Nqo@dZOKn*^kmf(`SFJp&`iz7V9IDV9M5|oQ~e}15eS-qcj2pP%fqN>E7 z04sBA00KG(!{B5ZG9w;FX5A;9u<>nhABykNm~E2CEB}OKIdw^*%+i|1N5_it#I;qj zmfarD)%}yP^~0rq?$R2^@5T?7kHAZKe_U$y(e3r|mH4@?-JQjwp@=!$IeyV%b+Myu ze`$HpduRJ-b$ev-6P*8jy^7M+H((vnhe9u0@H8BO zKLr`H@faXz@)#9ptyLffp%0R{SFH_Abs>sE%oEniO;5WA4^(sAF4F&cEup@0{;ug% z+Y@t*dl4aPHSf|sA558a3|Cv5`&-)M&ho`cZtCU6z>!0rhmyb-r#YRu1q|x}^V2Ws zX=ajH(E-rC{g)K$@nB)*62G~as1(Rk$va%#SS&zRR2f$to60!@^ut+%zA%2{1WEquhEFDNX;NWZ=cK;WG}Zf} zv8hQx@iE3h{J#fqM=2>vJlCCE)rZ}2^aDBdPG@uB%6Iq@VZCVAO5aMcv6{z70pmKo z+fp7ZSuC4Xv^wuXgU=35=OFC-Sg1RnmX@BLa-RA52VG`&mdzR>hj7&9_|Jl?zRV07 z&G&!itL}E3*P&570Y4)&K4qauHe@ez^2`_iwO_|hAD6M26yh3^KSp*CekH7R`<~U_ z%Z<#*o&lG(pzYaYb}cGC3#s|xYU>dTTEA#a`LswuZqZJa@5(_^k;(q)6?`+0HmaB)S93Kmt1eNv~4x}uP*`<1a>XSrX8yg!ZviQT4 z-}S1ys{i5c*0}Xw6=BzYxhwm7Du+A|nWK=H_e9zAS8CF882uAtQ60o&dt1bciGD-8 ziqi1J#MP_~BGZ7)m-K1%w!?!pt9@lL1zjg6Cr&8|C@n2*8IvejwF_l8mAW%k+!6V4 zUHttaO}c{(c`#4@B?`=y^ZhxyFnYmnWuZRX66zhJ*jgeV-U@%DE`{MEf62hh8?>tk zELLpjBSA1Y9Fp+nO%GkiBFu+X$HLA|sM*KDxZSC)Tr>*DknX_lieI{Kbu3;6L+R=s z2*W&pIEjaomtD*+bxoGFRlEfGFUTnL#gBNhbN#@yhRTiYE-$OhZQKJ;bS9jMtzVdU zRn|F$#l^*wpwKQ5YidIw#!#^T>=%8%-m*JEU*C+5v{WQK9Q{I0PC|mdgtCT->WGI% z8yml-Q-fYg0iYkHkWDX-N)PKMupEMd?2^z&#n-0Bl+!kXH>X)at7hpbPeF75&NE2| zPRFFna*-bF)}RYD730%#R{m{0JT%FqK%o8?b0wD8qUKdFRy8C-s24%mT*yY_0Ab;- zyD3SGJ-xCg{Xk4qw5)7(@X1pv4d;zMm=~=Mmjr-7 zipKCIHqe^cGf3usGy#(SGZU%L?#~T_dlhAn)QBEOIvrLro4f08E-)Dx-3O3l{8IWF zfR(*~;DmIuq4ofFLw^#Cxl8a8utoeUTXLGQ$iz@aZLa8*`HxE~8bLO9X}A5K7+<6T z0{|3I3*ITj6x{ZbMqHMJ>I~$BUzUJX{*6yKQlnq?==XK=-cp!h$i)0OO@-@_Oe(BG z&ki`GX#EV5nOj;~#Je;Zfx#H60(~huJoG<=Gs&{vfI03gdiv?s4;Fn%S|KOmP}U$^2l+-`d#VVe4KlClBz)JpEO8>*l!=rvwOf1FV4p zEIa2NkV(Uq!hZBMvmGA&A4*E-*{SA+@l)6=beN#{jNcG`kb^jryGezKXT`@4ANHS$ z=?D{E|ERuoe)3vU*2M8bZU!QO(}-U!{NyuVBm>gGxc_bp&Y0EqG6&yi`TMlKw1HGJ zCr-|0W?NgfGdbb;-wqp$2YfWp;V6fifICfZ-vsQJiPxj*^Le58n|?U{b>#UU->Z+M zFEF6|p3u|R9ErnX=-*Xs4!r$y;s<38AMDGfSx#@x|1jR;9%?!KK|;U7)W#;_)#&dp zHOXKV20>w96VrJw()EU}J@KaynToj7(g74~MB4Mk3iX#P&$A=^+a3QEte+NLbjUR~!6LdNteJ)=jLWg;; z5T)mT|2|6;%FllT6GeGDf=66be~-M>5FTLxI_L1mQ(IH=(uGS)OXVhIPENXux%7gU zXqC8TP{N7s_aUaHI%E?K;_D2egfxYYO&=*HOyDb8TGAUE^bL_U(GXhzdDQdMm$|y1~H`#KTiK+&q=0+`N-VPo^~fIudnxCPGx z79Xh|7d9b_W$QS}WctDA;oS9X@;ahqg}q4B_)Z$3S_`1HnUd> z7WM~X3$b+n?k9xlu5qo{#Sp(sNJ#9bz`5#NI{lpgLZVAX99Z+6mW22n?CtDYsFD^w zm;lM)wbL#fuTA}i;ETO!ugF(EYDsm;3XtdLgIC! z3HuZ-`g|LJw*VomcIByqv$ONx&A$FD0TA|v3hrtoQQq>Bn{9wCSP!azh;xu{N8P|( zg*zhN2726Y4ZC6}sdD}VUlYMiWLuv}fqQ8LqJBdBuq+vxAPA3 z1OL1;k;vd_Z?~XpivrkbnT5;>3mqrV_py_pCLEn0iZx%ecS6fkC&a+kAl@~)pg`z{9R#m~ zr+Hy=gh1fz8DW!;i!~lj!Xb1@?5>UF9H=R5E;G`uz1pR8ulz-URg$0)tK8L#rO+R} zgW6V@+b;u;ybSP4R{sIG|F)`*!=p|0CR5z>Ky>cPOOob=ZQ1z%yRtbw;i>E&Rb@TC z;q-&4;eW=C?Nr{pq&KTX!b&6o4z>R-&00Bm3e~Wjn$I?GSG$7L(9nw3%69X9{qG;YP(ADhvmS&^ifMbKCqmE> zVGOeIY=CDTu!1y#;`FqIL-G&^^a!c#Ht&OAZ0>!;I^Egh$B!c)`+!i0$FM72KSbBy zzZeRk)gUdx5x2&!7w4p+4EP42;2V^!xdJ`>VX5iiRIe#Lp=kl~B$2A$-535zgFi9v5HM$7>9N$3a`N!`T zyFI7z*g|(8jq`P7NLb*;ME#KEyQPWiSg;J*gzlCsAdZh^vxFO-UX@Ue4nhZ0{Dozr zi-D|)JqJVp{FZ9 zYC}K${DK;}KP~G6>ujsv+a!&=M>vgD(E7p>=3RtspFDW6IqsH5d=hO?#NBSIVd& zAnklp7L6Q^z7DurtO~v|1WM426_2B_77MEhgm_ixNV?8th4lXJd#X1? zwMl%ce5hVZ`r6DgD)%C~k{)DX6xp_4fax9@Vmk0tGIMK5Sf8Ld4&I(=+EO0U9DDC6 zU-%O(%niG+&$Z{#cJm&+xN6BZ5Y0yS$&*zBeJ-Fu_F(duD|g8$Rv?xL!8A%~0)ma{ z3x7+J5AOXW+|*k4g`;$*-{1bs4Vq6=F?-M+nU8uFcKnZav-7|8nI?97Z1a^5U1|MB zD4`uETTGW1y)t@jW)Uk}jMBSW21J$!9UxOTPt!{2#n&n1^E$8ShPpJSUL_#L@b!)5 z*Z-}cgf*~wUcW>;8zc8-wbadq)7x_8ynrzZ>C_A4F#hr<51TM{W#DbNLJheYjG%jS z#Y-1^#G`*CuSfa>(dyZtgG=j_+4q%tb=|w!N#Gw;s=mR@c$3Dm78)|MzDBzwzl_WS zmBfUS=7aBjwGrNvil2?+OgiYlk~=j&$;+*@%&CWWxCZvQws22dhoUKslY2|60RGtU zi@}A{7`7OV3S^h>7kV{+g)Sk9Q-M9w#<)CwjZ@A^OvSJ`~~Ip*vlhMiw)$GBYBUl!P1cFS=URV4i|WL|1rw9?w$2a$SamSVs3B+K8sWN zcf-?=T4>NvD;n4LNj^g#_XY)EnO)Edw)0NM6YH}ft)sMmey%tpNf&XP7C>SWIF3|6RrEVv#i9j2)Tv9HFJ-1|&0E>CB?uDG;-~$XH(Ua!`hWhc7xXgTovy z@&XYMu(;AlK3iutf_>u-c+Rn_bQ3u#?;PU|ASxkN{g$v?!&kTzm1MyxNM;&?`78sU z($hBzM2-w?y$XUVWY0lGK0(arMY`QZV~OWmYID)oK?wKd3=ucIZd|>{iCE{E1~QW<#hfbO$n3~mJ6@ochWP%dcLzO)X)eFXuU$733<7&aP`=QhBoa2%rK2# zDHBS*wp<`GkK$K<8rO$xWvXw>%&@3#> z&1H~swclqCF@S>9TH1?q)&+`hyF_`M_)ICfdpB2BRDd8lxq7T9im9@c`RptM8pMba zFWGb#5f!r&a`?F{CE1H2G)zEjeIEiz@HdDs)KyW~PNTCkiS%%k3KJ{4WSIF_zrS|K zmg{6nB5kf2kVNR4sQs`DKX`E5O-q(CB(^W*3m%l`5F<87G^9`k&ztvb4~LqQc6ncT8no_ z&QzHJR3t@VFnOAr^*YFq(gcBG6^BQjyeVJuu-~!y-rL*VW)~P3*wn-^t!{i!LcFpr z6*znYr9j7b@gnCq2Pfx!D%Y+(Z>j5^7x%sb^PRi6=+v=FYEz`Pw8ZzfB6vfb)GO)&=%Z5 zT1c$V;65}_@2zB)GB3namx`T|&~?zy((B;$e%qiJ(97Q+I=brW>J-p2RNTA$c`_t) zdzSRQj%n+9`<2IgE+FF8&!|oc`HvaWFf{b6^??0V$;o(4EL|tb)?TMkc|Fl?i1|6; z31N%DU-a)Y&$iaq)|qr;;|~@e%#DqC8YiO2*|Ke{*t%PQ7gi<7uJY@FN$({do|Ua1 z19^0)1}6GGK{mm=pWHrmrM!I{=>sO7&&}CM<-N05@uMu%*V5 zfxh*$if%`|X%KWV3#&7X$;w*Vkz4=V@i6LCj>uv+g*&_55iSVac_fQfG4?>gpe`7V zz|Ap>&LWin=^`dS`Qab*(b@;-6PX{#u!gecX1OQ_^EcPHUDKh#+lfqV*Lyh=|Sc>q5^o9*kP{iP`^ZDn0w!7IY|6H{}q0a$kh1xGv)?}e68#%c={73y7$wAF}YHDl^5pi`^Q^CQACia*`^NS z6Nc`=5228t%a>!?-9)5bM@Q`1o19>|2w;U$A0ftJv)0tdGmdj!Hr$TS34dD$q_fTyguKjK_6@j zvEtZ>C=VxZ1Y+_lUQ{@O*hL2hg*Kti=TeJ5z?U$&9K{9jgC~16Vc}bClas-Uf*42~ z@U&W_TtrMmLnG{OD>@K_MoUKg7^0uKftZovH^Ry>CLj}c&()qd+@62UHW>Lyw6Z*E z!>UIGUn^2xDw40H!0xC6R0ReQ`>$lK59yrM32T4Xi0|V9#mRY4V1RIusEUe;2iAp& zsmel37lk&$s*2sXtuh+rM|^PZMsRi;aeq|wbnT?*8kHj`j1^T#H;L3lmyky*Xc|O&=WfZY>ZS|dwj=H**<@TualkGoP;DQ%euO-~h&$Xuql0@LE4l&7y zhK_)EVJh10cS(t#DEh;gr5)x29SZ6ElpaJc_afo%_laRKV2{Aly^%0h#iukJi$=3= z2BHTTR7uM>75OL;cPLs6Sk02;Z;(M>c50X{N>M?m6lxbeNto<<{?_3Q2ykibl7eKh zA-j(}53G0ovV*E2!7U~o4}U_YwHN^@p~#l~M6aX-t2jfg`e(-fpieu_9F95bO5Cp@S{m3$nPJBWC-V>nsNvAmBTv$Ml#WMyT`Xij|(A9y_bkH8GJ zXGc4;?4m!BV83HWYd{y!QYD;zV~W~C8SDQobcXCN8ob4T5#UU{;2kt1!|3D@*wPnM z2C~sgJ{$q9eG$)BNe*|9VH*x!RU^sVqoLuVCRWg*Mw@6=F%{wOk%MA@-&qeIb{wM~ zMb>OYv+YS4zM*^hnX~i8AVU>N=;$+={9N`eR5PYYHIN#SXPyjg<|yNcd50rw&GmzF{eTZj>A%i;$Et3na~(+8cd)1P!^eQg9z5RiXTFWQsS{EHH5g1+upa*c{Rr8B+sqlw{eb6DdnXoxh*b@Fma_@>Ob49iK z@Y`|&CfdT%0lFG8rZ3$P-Wn+8%TWT#MO9xSrZufD&ja%FGl6K9NI;9?kckn0|~= zvsjb=fi9P~-|OY8UjqZr;$LZ_^Q5>fEiGqfBRC?BcwC;L>5cr(nKf_*Aji`5mtX$uOb5pAsBSmcz3|xIKzq?y2OSKkuLWVC?oQ z#oJ$DC|_v9i1I5f1a9D~YxF0|bLJ$H^?2G6Wo*|~N77z+T=7h0Yb3>S1l@k0O;^O} z!l+T83HCQG(&WeF`#X2qf_EJ$cYNEPxXY$KIvP}gaEoI1uXc)ameBDkCGBF^NGgeQ zeM1$uiIF5DumBViQ3JYoAYuc8vOK27Ih2lE9pA6_uTD>bD?zeAFNF3~Fk1cSBynZW zHTB-h0Mg~jdVSI^4i#3}a53IcA2}k+P3_2ld}k8ur?g8gy*0EB*p&c2`|s9+!R zi{PeSsE?uACk?$njFK1REfiM1n>8zP&y=%7R+3#G2`B=f653oQ6@8MdC~!l(Oi0PA zC#+A{v4pw522w@*dybRuFT~hGV_R) z8;x5w?usehYku}P(!9BrGN~1w!ixHYu6IAym3}C5DR%9s+9j8!Po?s#V2$yS;R^}% zqutZ_`SX)M1vr7&)YSD^<2Wl^Pr9Qc2Hw(!?-7?a5C=P>>*`?@~a zVA#dhw)bdvhGI({nGh!0$Xl-^0cE}{&|r-G{QLmo$9`VI4_?Rjks^<~F$Eh##hS)K zKz>1#sa@$ewC4EG5Bmx(XAaGlX)C`E=st_XgPCN4NNg`t^Y;qrh`;TIfR;P_6AoWM zA)n^#`ol)BSYHrc9SxfOhB=xC0kTQSdC+>JwKY7g>G5;u&+fOno4TNN3sb$bn8@`~ z`))@NWxIeC|>EP$*zJ^}~b!!-jO{gMd z05_uVDDq7T62Ct7?OVAQdMo#RHB!F55r>t}zuaOZ%)oH+uPY}`|7<#%HW#Y|22>K$ zcn~N~5LijL`%e-Y>djzA@za{iQT$6Y%ytRIFx-UxxI11jGJ>x{G%X*YGKqBiSAFLqP8is8+&S?IHrz2Nh}oTPc&l*G!_l`Py8eOkIE0?nV;8$R90E zp*dp|K6S^ZDckE0LpM{SIT7q{-g!@W)=|KQK?7ld+ zs_W%f;K}a8cWez8z+fCA={_qgvD0e@5$EY^9)0oG&!1Yi;JX=LtRdBrBKfx%N=v?! zw3KjvtCEdbm?wYd^1%1sf~yhzZpT?Pom`8<-S++&zyp?vU-H$Kzo3kI2ZzC68NXbv zI)D@)B`GdLKWRvT@g3xL5P#eoE>*6pTvQw$1Pjb`Qr>?KV<&|~W$*|@oyB%u9!*^y zk@lu`&{vy9P9fo`g3f4UNg%`K!rJdFsV$eGp?V1EcNcjaUDn~mKraac5f+$CEm_Jf z5~31{VtW9%X-gW@@)o{NC?t{n&M(~#h0z3MofJz!fG@=ObPbNy$iQGQ>GC&el_8c$ zwtz{LrIS;k$x0OH!0pI^fI9rIO{D3kD_h*AH}%9sUg>ap16x{W58}$BgTMNjM%B*A zgn}Val-5M27@rc1Jqc{&<7GSIVshOjBb~5t&sV@^k{};B(WMCnCs3VN+h^}FNns-v zM3&KLG&WjBYQAr1C~@j_V=ZwED5>_J??^=tmAMQjXb!tLz7?HMv#EbkTo-w}Lx z8v4Da=3)=B$E1Dvd&3LDLjENqF}|gn>)o|qucxZX8T2v2j4Y}MZtKZm(CVSV{{COT z?gOC)2G2s_8nI!yIfn-aR>?#JcchN?vi8?w(*g2l4iI}|o=7%W535Ny7@$0B4yMxq zSVlM2vyMucJyu(kNe^mDFlyL9QG5I4Qu64tb6wqLU&4`LR&O zoKv0aF^(mY?P9_nV&HvrljSNv-F|T!wX%Mm3wd_!dEoFD>H4GyZOF>lLgMYp%ea3( z_CCygPJnOUgwV1?9#61-i(5ld<~=dn)juks)y1{ZW?+S=jww+=;D|k@ti7ykeEb&M znP9GVGpD3q>41BObs}4;0P*4lWzh(-dVa||B_)Z7 zFxUS2ve#&6@5vq4Xpx+}h4x+os{#IBqrkN!dh1HF1$rfAfS4v4Qp{`C>=ZG4Pby7UYO@*`}p^8 z`hzguk1ax@pHsdLy;@%XZ0Hy!l5>vFh`;hao>h!|{_9#QTGWyS8~wYZiwsyY02r8z zc%LWXG&}^iUeJx>ThCuG7!@eC!DHW{zoRDx3$O_DQARL8RN(_#k}wf52Mgt)BE@P% z_VvB7*-%mUA-ZW4y}}DDR1jpA8}mqIZ0a>+{oHsy?HMu}MT#NP>mwOEi|cl*zvQt5 zlkb{Nq$)`GpZGxcE^-JUR{k?y`Lmd#$fvyn&;1m~4>K~24}EbYR?5i@dV}zHy)`0r zd37-QpBwcM$m!Miw!Xfcw@4b2!1Ba|63^vyLt{j7)E)7@Z;Yae3$=l4f;FbLE}la7o^DW{>aFG z#yWwvc;dBT7NA0e3N!85L0vhSm_Pi&XMxMwQ9XY<*qDRR-wt4`#S~)OZpR<0$1G%f zJTFN1SwEWSuqa@vM%-R53!cgF5)=U|2!qy7k2HJdpZDZjw2jPzm@}qFzSoL&*EzR! zpw)u@O)CaTF-s!9wtx7$zKe!BA$8-H_^$X&Q@7Jf^%BzrVs5eg;ClQmdiBu5SMgaj zRymX-j+Nrv8QDW-N@Nz8LArs(k*_VrkxlUymmrRBnFrx_RZj&K?~(||=OUWZB(F+* zlk~I(rEczbYa={m_*|CF^!u*| { const actual = await vi.importActual("../networking"); + const { readFileSync } = await import("node:fs"); + const { resolve } = await import("node:path"); + const metadata = JSON.parse( + readFileSync( + resolve(__dirname, "../../../../../litellm/proxy/public_endpoints/provider_create_fields.json"), + "utf8", + ), + ) as { provider: string }[]; return { ...actual, getProviderCreateMetadata: vi.fn().mockResolvedValue([ + ...metadata.filter((provider) => provider.provider === "BIGMODEL"), { provider: "OpenAI", provider_display_name: Providers.OpenAI, @@ -157,6 +166,39 @@ const chooseProvider = async (user: ReturnType, provider await user.click(options.find((option) => option.textContent === provider) ?? options[0]); }; +describe("CredentialModal with BigModel metadata", () => { + it("requires a masked API key and shows the default base", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal(); + await chooseProvider(user, Providers.BIGMODEL); + expect(await screen.findByLabelText("API Base")).toHaveValue("https://open.bigmodel.cn"); + expect(screen.getByLabelText("BigModel (Zhipu AI) API Key")).toHaveAttribute("type", "password"); + fill("Credential Name:", "bigmodel-key"); + await user.click(screen.getByRole("button", { name: "Add Credential" })); + expect(await screen.findByText("Required")).toBeInTheDocument(); + expect(onSubmit).not.toHaveBeenCalled(); + }); + + it("saves the key and an explicit base for BigModel", async () => { + const user = userEvent.setup(); + const onSubmit = renderModal(); + await chooseProvider(user, Providers.BIGMODEL); + await screen.findByLabelText("BigModel (Zhipu AI) API Key"); + fill("Credential Name:", "bigmodel-key"); + fill("BigModel (Zhipu AI) API Key", "synthetic-bigmodel-key"); + fill("API Base", "https://bigmodel-proxy.example"); + await user.click(screen.getByRole("button", { name: "Add Credential" })); + expect(onSubmit).toHaveBeenCalledWith( + { + credential_name: "bigmodel-key", + custom_llm_provider: "BIGMODEL", + credential_values: { api_key: "synthetic-bigmodel-key", api_base: "https://bigmodel-proxy.example" }, + }, + [], + ); + }); +}); + describe("CredentialModal with Anthropic workload identity federation", () => { it("deletes stored exchange fields when switching to a static token", async () => { const user = userEvent.setup(); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 0ebd2cdff17..51a248daafa 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -20,6 +20,18 @@ const BUNDLED_MODEL_MAP: Record = JSON.pa ); describe("provider_info_helpers", () => { + it("resolves BigModel metadata, name, logo and model placeholder", () => { + const metadata = PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "BIGMODEL"); + expect(provider_map.BIGMODEL).toBe(metadata?.litellm_provider); + expect(getProviderLogoAndName("bigmodel")).toEqual({ + displayName: "BigModel (Zhipu AI)", + logo: providerLogoMap[Providers.BIGMODEL], + }); + expect(getProviderLogoAndName("BIGMODEL")).toEqual(getProviderLogoAndName("bigmodel")); + expect(providerLogoMap[Providers.BIGMODEL]).toContain("bigmodel"); + expect(getPlaceholder(Providers.BIGMODEL)).toMatch(/^bigmodel\//); + }); + it("maps Microsoft 365 Copilot to its chat model provider and placeholder", () => { expect(provider_map.MICROSOFT_365_COPILOT).toBe("microsoft_365_copilot"); expect(getPlaceholder(Providers.MICROSOFT_365_COPILOT)).toBe("microsoft_365_copilot/chat"); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 2541aa45050..6bda9b4fccb 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -6,6 +6,7 @@ import anthropicLogo from "../../public/assets/logos/anthropic.svg"; import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png"; import basetenLogo from "../../public/assets/logos/baseten.svg"; import bedrockLogo from "../../public/assets/logos/bedrock.svg"; +import bigmodelLogo from "../../public/assets/logos/bigmodel.png"; import cerebrasLogo from "../../public/assets/logos/cerebras.svg"; import cloudflareLogo from "../../public/assets/logos/cloudflare.svg"; import cohereLogo from "../../public/assets/logos/cohere.svg"; @@ -197,6 +198,7 @@ export enum Providers { xAI = "xAI", XINFERENCE = "Xinference", ZAI = "Z.AI (Zhipu AI)", + BIGMODEL = "BigModel (Zhipu AI)", } export const provider_map: Record = { @@ -325,6 +327,7 @@ export const provider_map: Record = { xAI: "xai", XINFERENCE: "xinference", ZAI: "zai", + BIGMODEL: "bigmodel", }; const standaloneSubproviderSlugs = new Set(["bedrock_mantle"]); @@ -333,6 +336,7 @@ export const resolveLitellmProviderSlug = (provider: string): string => provider_map[provider] ?? provider.toLowerCase(); export const providerLogoMap: Partial> = { + [Providers.BIGMODEL]: bigmodelLogo.src, [Providers.A2A_Agent]: a2aAgentLogo.src, [Providers.AI21]: ai21Logo.src, [Providers.AI21_CHAT]: ai21Logo.src, @@ -497,6 +501,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Voyage]: "voyage/", [Providers.WATSONX]: "watsonx/ibm/granite-3-3-8b-instruct", [Providers.ZAI]: "zai/glm-4.5", + [Providers.BIGMODEL]: "bigmodel/glm-5.3", }; export const getPlaceholder = (selectedProvider: string): string => {