mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(zai): keep BigModel's json_schema as sent and never drop the caller's max_tokens under drop_params
This commit is contained in:
parent
80912bba61
commit
20c90d4ed1
4 changed files with 65 additions and 5 deletions
|
|
@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig):
|
|||
def custom_llm_provider(self) -> str:
|
||||
return "bigmodel"
|
||||
|
||||
@property
|
||||
def prompts_unsupported_json_schema(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
|
||||
return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,10 @@ class ZAIChatConfig(OpenAIGPTConfig):
|
|||
def custom_llm_provider(self) -> str | None:
|
||||
return "zai"
|
||||
|
||||
@property
|
||||
def prompts_unsupported_json_schema(self) -> bool:
|
||||
return True
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: str | None, api_key: str | None
|
||||
) -> tuple[str | None, str | None]:
|
||||
|
|
@ -103,14 +107,15 @@ class ZAIChatConfig(OpenAIGPTConfig):
|
|||
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params}
|
||||
max_tokens: Final = sent.get("max_completion_tokens", sent.get("max_tokens"))
|
||||
drops_max_tokens: Final = max_tokens is None or (
|
||||
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_tokens)
|
||||
max_completion_tokens: Final = sent.get("max_completion_tokens")
|
||||
falls_back_to_max_tokens: Final = max_completion_tokens is None or (
|
||||
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_completion_tokens)
|
||||
)
|
||||
max_tokens: Final = sent.get("max_tokens") if falls_back_to_max_tokens else max_completion_tokens
|
||||
return {
|
||||
**optional_params,
|
||||
**{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS},
|
||||
**({} if drops_max_tokens else {"max_tokens": max_tokens}),
|
||||
**({} if max_tokens is None else {"max_tokens": max_tokens}),
|
||||
}
|
||||
|
||||
def transform_request(
|
||||
|
|
@ -121,7 +126,11 @@ class ZAIChatConfig(OpenAIGPTConfig):
|
|||
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
|
||||
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
|
||||
) -> dict[str, object]: # mutable-ok: inherited contract returns the request body
|
||||
schema: Final = _schema_without_native_support(model, optional_params.get("response_format"))
|
||||
schema: Final = (
|
||||
_schema_without_native_support(model, optional_params.get("response_format"))
|
||||
if self.prompts_unsupported_json_schema
|
||||
else None
|
||||
)
|
||||
schema_message: Final = (
|
||||
()
|
||||
if schema is None
|
||||
|
|
|
|||
|
|
@ -197,6 +197,29 @@ def test_chat_does_not_send_the_openai_organization_to_bigmodel(monkeypatch: pyt
|
|||
assert "openai-organization" not in route.calls.last.request.headers
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_chat_sends_json_schema_response_format_as_given() -> None:
|
||||
route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond(
|
||||
json={
|
||||
"id": "chat_test",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "glm-4.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "{}"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "Ann likes 7"}]
|
||||
response_format: Final = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"n": {"type": "integer"}}}},
|
||||
}
|
||||
_ = litellm.completion(model="bigmodel/glm-4.6", messages=messages, response_format=response_format)
|
||||
body: Final = json.loads(route.calls.last.request.content)
|
||||
assert body["response_format"] == response_format
|
||||
assert body["messages"] == messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", (False, True))
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
|
|
|
|||
|
|
@ -164,6 +164,30 @@ def test_zai_drop_params_drops_max_completion_tokens_above_model_limit(
|
|||
assert body.get("max_tokens") == (requested if forwarded else None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_over_limit_max_completion_tokens", [False, True])
|
||||
def test_zai_drop_params_keeps_max_tokens_above_model_limit(
|
||||
local_model_cost_map, respx_mock, zai_response, monkeypatch, with_over_limit_max_completion_tokens
|
||||
):
|
||||
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
|
||||
over_limit = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + 1
|
||||
over_limit_max_completion_tokens = (
|
||||
{"max_completion_tokens": over_limit + 1} if with_over_limit_max_completion_tokens else {}
|
||||
)
|
||||
|
||||
completion(
|
||||
model="zai/glm-4.5-flash",
|
||||
messages=[{"role": "user", "content": "Count"}],
|
||||
max_tokens=over_limit,
|
||||
drop_params=True,
|
||||
**over_limit_max_completion_tokens,
|
||||
)
|
||||
|
||||
body = json.loads(respx_mock.calls[0].request.content)
|
||||
assert body["max_tokens"] == over_limit
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}])
|
||||
def test_zai_forwards_json_schema_to_models_that_enforce_it(
|
||||
local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue