fix(zai): keep BigModel's json_schema as sent and never drop the caller's max_tokens under drop_params

This commit is contained in:
mateo-berri 2026-10-29 14:04:15 -07:00
parent 80912bba61
commit 20c90d4ed1
4 changed files with 65 additions and 5 deletions

View file

@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
def custom_llm_provider(self) -> str:
return "bigmodel"
@property
def prompts_unsupported_json_schema(self) -> bool:
return False
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)

View file

@ -45,6 +45,10 @@ class ZAIChatConfig(OpenAIGPTConfig):
def custom_llm_provider(self) -> str | None:
return "zai"
@property
def prompts_unsupported_json_schema(self) -> bool:
return True
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
@ -103,14 +107,15 @@ class ZAIChatConfig(OpenAIGPTConfig):
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
supported_params: Final = self.get_supported_openai_params(model)
sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params}
max_tokens: Final = sent.get("max_completion_tokens", sent.get("max_tokens"))
drops_max_tokens: Final = max_tokens is None or (
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_tokens)
max_completion_tokens: Final = sent.get("max_completion_tokens")
falls_back_to_max_tokens: Final = max_completion_tokens is None or (
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_completion_tokens)
)
max_tokens: Final = sent.get("max_tokens") if falls_back_to_max_tokens else max_completion_tokens
return {
**optional_params,
**{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS},
**({} if drops_max_tokens else {"max_tokens": max_tokens}),
**({} if max_tokens is None else {"max_tokens": max_tokens}),
}
def transform_request(
@ -121,7 +126,11 @@ class ZAIChatConfig(OpenAIGPTConfig):
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
) -> dict[str, object]: # mutable-ok: inherited contract returns the request body
schema: Final = _schema_without_native_support(model, optional_params.get("response_format"))
schema: Final = (
_schema_without_native_support(model, optional_params.get("response_format"))
if self.prompts_unsupported_json_schema
else None
)
schema_message: Final = (
()
if schema is None

View file

@ -197,6 +197,29 @@ def test_chat_does_not_send_the_openai_organization_to_bigmodel(monkeypatch: pyt
assert "openai-organization" not in route.calls.last.request.headers
@respx.mock
def test_chat_sends_json_schema_response_format_as_given() -> None:
route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond(
json={
"id": "chat_test",
"object": "chat.completion",
"created": 1,
"model": "glm-4.6",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "{}"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
)
messages: Final = [{"role": "user", "content": "Ann likes 7"}]
response_format: Final = {
"type": "json_schema",
"json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"n": {"type": "integer"}}}},
}
_ = litellm.completion(model="bigmodel/glm-4.6", messages=messages, response_format=response_format)
body: Final = json.loads(route.calls.last.request.content)
assert body["response_format"] == response_format
assert body["messages"] == messages
@pytest.mark.parametrize("is_async", (False, True))
@pytest.mark.asyncio
@respx.mock

View file

@ -164,6 +164,30 @@ def test_zai_drop_params_drops_max_completion_tokens_above_model_limit(
assert body.get("max_tokens") == (requested if forwarded else None)
@pytest.mark.parametrize("with_over_limit_max_completion_tokens", [False, True])
def test_zai_drop_params_keeps_max_tokens_above_model_limit(
local_model_cost_map, respx_mock, zai_response, monkeypatch, with_over_limit_max_completion_tokens
):
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
over_limit = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + 1
over_limit_max_completion_tokens = (
{"max_completion_tokens": over_limit + 1} if with_over_limit_max_completion_tokens else {}
)
completion(
model="zai/glm-4.5-flash",
messages=[{"role": "user", "content": "Count"}],
max_tokens=over_limit,
drop_params=True,
**over_limit_max_completion_tokens,
)
body = json.loads(respx_mock.calls[0].request.content)
assert body["max_tokens"] == over_limit
@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}])
def test_zai_forwards_json_schema_to_models_that_enforce_it(
local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through