fix(zai): follow json_schema natively on glm-5.3, prompt other models, drop over-limit max_tokens under drop_params

This commit is contained in:
mateo-berri 2026-10-29 11:41:07 -07:00
parent db34951d38
commit 80912bba61
5 changed files with 134 additions and 30 deletions

View file

@ -5878,30 +5878,28 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
Returns the prompt str that's passed to the model as a user message
"""
custom_prompt_details: dict | None = None
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict[
f"{model}/response_schema_prompt"
] # allow user to define custom response schema prompt by model
elif "response_schema_prompt" in litellm.custom_prompt_dict:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
model_prompt_key: Final = f"{model}/response_schema_prompt"
custom_prompt_details: Final = prompts.get(
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
)
if custom_prompt_details is None:
return default_response_schema_prompt(response_schema=response_schema)
return render_custom_prompt_template(
template=custom_prompt_details,
messages=[{"role": "user", "content": f"{response_schema}"}],
model=model,
llm_provider=llm_provider,
)
def default_response_schema_prompt(response_schema: dict) -> str:
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
"""
Used if provider/model doesn't support 'response_schema' param.

View file

@ -1,13 +1,43 @@
from collections.abc import Mapping
from typing import Final
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4"
TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"})
def _is_json_object(
value: object,
) -> TypeIs[Mapping[str, object]]: # guard-ok: cost-map rows and request params are JSON
return isinstance(value, Mapping)
def _json_field(value: object, key: str) -> object:
return value.get(key) if _is_json_object(value) else None
def _exceeds_output_limit(model: str, max_tokens: object) -> bool:
info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai")
limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens")
return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit
def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None:
if _json_field(response_format, "type") != "json_schema":
return None
if litellm.supports_response_schema(model=model, custom_llm_provider="zai"):
return None
schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema")
return schema if _is_json_object(schema) else None
class ZAIChatConfig(OpenAIGPTConfig):
@ -72,11 +102,39 @@ class ZAIChatConfig(OpenAIGPTConfig):
drop_params: bool,
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
supported_params: Final = self.get_supported_openai_params(model)
sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params}
max_tokens: Final = sent.get("max_completion_tokens", sent.get("max_tokens"))
drops_max_tokens: Final = max_tokens is None or (
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_tokens)
)
return {
**optional_params,
**{
("max_tokens" if param == "max_completion_tokens" else param): value
for param, value in non_default_params.items()
if param in supported_params
},
**{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS},
**({} if drops_max_tokens else {"max_tokens": max_tokens}),
}
def transform_request(
self,
model: str,
messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature
optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
) -> dict[str, object]: # mutable-ok: inherited contract returns the request body
schema: Final = _schema_without_native_support(model, optional_params.get("response_format"))
schema_message: Final = (
()
if schema is None
else (
ChatCompletionUserMessage(
role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai")
),
)
)
return super().transform_request(
model,
[*messages, *schema_message],
optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}},
litellm_params,
headers,
)

View file

@ -54308,6 +54308,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"zai/glm-5.3-flash": {
@ -54322,6 +54323,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing",
"supports_vision": true

View file

@ -54308,6 +54308,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"zai/glm-5.3-flash": {
@ -54322,6 +54323,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing",
"supports_vision": true

View file

@ -10,11 +10,15 @@ import pytest
import litellm
from litellm import completion
from litellm.cost_calculator import cost_per_token
from litellm.llms.zai.chat.transformation import ZAIChatConfig
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
JSON_SCHEMA_RESPONSE_FORMAT = {
"type": "json_schema",
"json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"name": {"type": "string"}}}},
"json_schema": {
"name": "profile",
"schema": {"type": "object", "properties": {"favorite_number": {"type": "integer"}}},
},
}
@ -130,32 +134,72 @@ def test_zai_sends_response_format_and_maps_max_completion_tokens(respx_mock, za
"token_limits",
[{"max_tokens": 300, "max_completion_tokens": 15}, {"max_completion_tokens": 15, "max_tokens": 300}],
)
def test_zai_max_completion_tokens_wins_over_max_tokens(respx_mock, zai_response, monkeypatch, token_limits):
def test_zai_max_completion_tokens_wins_over_max_tokens(token_limits):
mapped = ZAIChatConfig().map_openai_params(token_limits, {}, "glm-5.3", drop_params=False)
assert mapped["max_tokens"] == 15
assert "max_completion_tokens" not in mapped
@pytest.mark.parametrize(
("over_limit_by", "drop_params", "forwarded"),
[(1, True, False), (1, False, True), (0, True, True)],
)
def test_zai_drop_params_drops_max_completion_tokens_above_model_limit(
local_model_cost_map, respx_mock, zai_response, monkeypatch, over_limit_by, drop_params, forwarded
):
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
requested = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + over_limit_by
completion(model="zai/glm-5.3", messages=[{"role": "user", "content": "Count"}], **token_limits)
completion(
model="zai/glm-4.5-flash",
messages=[{"role": "user", "content": "Count"}],
max_completion_tokens=requested,
drop_params=drop_params,
)
body = json.loads(respx_mock.calls[0].request.content)
assert body["max_tokens"] == 15
assert body.get("max_tokens") == (requested if forwarded else None)
@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}])
def test_zai_forwards_json_schema_response_format_as_sent(respx_mock, zai_response, monkeypatch, pass_through):
def test_zai_forwards_json_schema_to_models_that_enforce_it(
local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through
):
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
messages = [{"role": "user", "content": "Ann likes 7"}]
completion(model="zai/glm-5.3", messages=messages, response_format=JSON_SCHEMA_RESPONSE_FORMAT, **pass_through)
body = json.loads(respx_mock.calls[0].request.content)
assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
assert body["messages"] == messages
@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}])
def test_zai_puts_json_schema_in_prompt_for_models_that_ignore_it(
local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through
):
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
completion(
model="zai/glm-5.3",
messages=[{"role": "user", "content": "Ann is 30"}],
model="zai/glm-4.6",
messages=[{"role": "user", "content": "Ann likes 7"}],
response_format=JSON_SCHEMA_RESPONSE_FORMAT,
**pass_through,
)
body = json.loads(respx_mock.calls[0].request.content)
assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
assert body["response_format"] == {"type": "json_object"}
assert body["messages"][0] == {"role": "user", "content": "Ann likes 7"}
assert body["messages"][-1]["role"] == "user"
assert "favorite_number" in body["messages"][-1]["content"]
def test_zai_glm_5v_turbo_bills_cached_input_at_cache_read_price(local_model_cost_map: None) -> None: