diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..14f54f99629 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5878,30 +5878,28 @@ def function_call_prompt( ] -def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str: +def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str: """ Decides if a user-defined custom prompt or default needs to be used Returns the prompt str that's passed to the model as a user message """ - custom_prompt_details: dict | None = None - response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}] - if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict: - custom_prompt_details = litellm.custom_prompt_dict[ - f"{model}/response_schema_prompt" - ] # allow user to define custom response schema prompt by model - elif "response_schema_prompt" in litellm.custom_prompt_dict: - custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"] - - if custom_prompt_details is not None: - return render_custom_prompt_template( - template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider - ) - else: + prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict + model_prompt_key: Final = f"{model}/response_schema_prompt" + custom_prompt_details: Final = prompts.get( + model_prompt_key if model_prompt_key in prompts else "response_schema_prompt" + ) + if custom_prompt_details is None: return default_response_schema_prompt(response_schema=response_schema) + return render_custom_prompt_template( + template=custom_prompt_details, + messages=[{"role": "user", "content": f"{response_schema}"}], + model=model, + llm_provider=llm_provider, + ) -def default_response_schema_prompt(response_schema: dict) -> str: +def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str: """ Used if provider/model doesn't support 'response_schema' param. diff --git a/litellm/llms/zai/chat/transformation.py b/litellm/llms/zai/chat/transformation.py index 3a72d233e6b..6b1fda43140 100644 --- a/litellm/llms/zai/chat/transformation.py +++ b/litellm/llms/zai/chat/transformation.py @@ -1,13 +1,43 @@ from collections.abc import Mapping from typing import Final +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 + import litellm +from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage +from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration from ...openai.chat.gpt_transformation import OpenAIGPTConfig ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4" +TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"}) + + +def _is_json_object( + value: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: cost-map rows and request params are JSON + return isinstance(value, Mapping) + + +def _json_field(value: object, key: str) -> object: + return value.get(key) if _is_json_object(value) else None + + +def _exceeds_output_limit(model: str, max_tokens: object) -> bool: + info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai") + limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens") + return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit + + +def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None: + if _json_field(response_format, "type") != "json_schema": + return None + if litellm.supports_response_schema(model=model, custom_llm_provider="zai"): + return None + schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema") + return schema if _is_json_object(schema) else None class ZAIChatConfig(OpenAIGPTConfig): @@ -72,11 +102,39 @@ class ZAIChatConfig(OpenAIGPTConfig): drop_params: bool, ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body supported_params: Final = self.get_supported_openai_params(model) + sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params} + max_tokens: Final = sent.get("max_completion_tokens", sent.get("max_tokens")) + drops_max_tokens: Final = max_tokens is None or ( + (litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_tokens) + ) return { **optional_params, - **{ - ("max_tokens" if param == "max_completion_tokens" else param): value - for param, value in non_default_params.items() - if param in supported_params - }, + **{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS}, + **({} if drops_max_tokens else {"max_tokens": max_tokens}), } + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature + optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature + ) -> dict[str, object]: # mutable-ok: inherited contract returns the request body + schema: Final = _schema_without_native_support(model, optional_params.get("response_format")) + schema_message: Final = ( + () + if schema is None + else ( + ChatCompletionUserMessage( + role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai") + ), + ) + ) + return super().transform_request( + model, + [*messages, *schema_message], + optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}}, + litellm_params, + headers, + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dbe0be24fdc..f432f92eb8a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -54308,6 +54308,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "zai/glm-5.3-flash": { @@ -54322,6 +54323,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dbe0be24fdc..f432f92eb8a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -54308,6 +54308,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "zai/glm-5.3-flash": { @@ -54322,6 +54323,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true diff --git a/tests/unit/llms/zai/test_zai_provider.py b/tests/unit/llms/zai/test_zai_provider.py index b3c9e8344a9..109b6b41e51 100644 --- a/tests/unit/llms/zai/test_zai_provider.py +++ b/tests/unit/llms/zai/test_zai_provider.py @@ -10,11 +10,15 @@ import pytest import litellm from litellm import completion from litellm.cost_calculator import cost_per_token +from litellm.llms.zai.chat.transformation import ZAIChatConfig from litellm.types.utils import PromptTokensDetailsWrapper, Usage JSON_SCHEMA_RESPONSE_FORMAT = { "type": "json_schema", - "json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"name": {"type": "string"}}}}, + "json_schema": { + "name": "profile", + "schema": {"type": "object", "properties": {"favorite_number": {"type": "integer"}}}, + }, } @@ -130,32 +134,72 @@ def test_zai_sends_response_format_and_maps_max_completion_tokens(respx_mock, za "token_limits", [{"max_tokens": 300, "max_completion_tokens": 15}, {"max_completion_tokens": 15, "max_tokens": 300}], ) -def test_zai_max_completion_tokens_wins_over_max_tokens(respx_mock, zai_response, monkeypatch, token_limits): +def test_zai_max_completion_tokens_wins_over_max_tokens(token_limits): + mapped = ZAIChatConfig().map_openai_params(token_limits, {}, "glm-5.3", drop_params=False) + + assert mapped["max_tokens"] == 15 + assert "max_completion_tokens" not in mapped + + +@pytest.mark.parametrize( + ("over_limit_by", "drop_params", "forwarded"), + [(1, True, False), (1, False, True), (0, True, True)], +) +def test_zai_drop_params_drops_max_completion_tokens_above_model_limit( + local_model_cost_map, respx_mock, zai_response, monkeypatch, over_limit_by, drop_params, forwarded +): monkeypatch.setenv("ZAI_API_KEY", "test-api-key") monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + requested = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + over_limit_by - completion(model="zai/glm-5.3", messages=[{"role": "user", "content": "Count"}], **token_limits) + completion( + model="zai/glm-4.5-flash", + messages=[{"role": "user", "content": "Count"}], + max_completion_tokens=requested, + drop_params=drop_params, + ) body = json.loads(respx_mock.calls[0].request.content) - assert body["max_tokens"] == 15 + assert body.get("max_tokens") == (requested if forwarded else None) @pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}]) -def test_zai_forwards_json_schema_response_format_as_sent(respx_mock, zai_response, monkeypatch, pass_through): +def test_zai_forwards_json_schema_to_models_that_enforce_it( + local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through +): + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + messages = [{"role": "user", "content": "Ann likes 7"}] + + completion(model="zai/glm-5.3", messages=messages, response_format=JSON_SCHEMA_RESPONSE_FORMAT, **pass_through) + + body = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT + assert body["messages"] == messages + + +@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}]) +def test_zai_puts_json_schema_in_prompt_for_models_that_ignore_it( + local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through +): monkeypatch.setenv("ZAI_API_KEY", "test-api-key") monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) completion( - model="zai/glm-5.3", - messages=[{"role": "user", "content": "Ann is 30"}], + model="zai/glm-4.6", + messages=[{"role": "user", "content": "Ann likes 7"}], response_format=JSON_SCHEMA_RESPONSE_FORMAT, **pass_through, ) body = json.loads(respx_mock.calls[0].request.content) - assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT + assert body["response_format"] == {"type": "json_object"} + assert body["messages"][0] == {"role": "user", "content": "Ann likes 7"} + assert body["messages"][-1]["role"] == "user" + assert "favorite_number" in body["messages"][-1]["content"] def test_zai_glm_5v_turbo_bills_cached_input_at_cache_read_price(local_model_cost_map: None) -> None: