mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(zai): follow json_schema natively on glm-5.3, prompt other models, drop over-limit max_tokens under drop_params
This commit is contained in:
parent
db34951d38
commit
80912bba61
5 changed files with 134 additions and 30 deletions
|
|
@ -5878,30 +5878,28 @@ def function_call_prompt(
|
|||
]
|
||||
|
||||
|
||||
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
|
||||
def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str:
|
||||
"""
|
||||
Decides if a user-defined custom prompt or default needs to be used
|
||||
|
||||
Returns the prompt str that's passed to the model as a user message
|
||||
"""
|
||||
custom_prompt_details: dict | None = None
|
||||
response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}]
|
||||
if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict:
|
||||
custom_prompt_details = litellm.custom_prompt_dict[
|
||||
f"{model}/response_schema_prompt"
|
||||
] # allow user to define custom response schema prompt by model
|
||||
elif "response_schema_prompt" in litellm.custom_prompt_dict:
|
||||
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
|
||||
|
||||
if custom_prompt_details is not None:
|
||||
return render_custom_prompt_template(
|
||||
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
|
||||
)
|
||||
else:
|
||||
prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict
|
||||
model_prompt_key: Final = f"{model}/response_schema_prompt"
|
||||
custom_prompt_details: Final = prompts.get(
|
||||
model_prompt_key if model_prompt_key in prompts else "response_schema_prompt"
|
||||
)
|
||||
if custom_prompt_details is None:
|
||||
return default_response_schema_prompt(response_schema=response_schema)
|
||||
return render_custom_prompt_template(
|
||||
template=custom_prompt_details,
|
||||
messages=[{"role": "user", "content": f"{response_schema}"}],
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
|
||||
def default_response_schema_prompt(response_schema: dict) -> str:
|
||||
def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Used if provider/model doesn't support 'response_schema' param.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,43 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage
|
||||
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
|
||||
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4"
|
||||
TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"})
|
||||
|
||||
|
||||
def _is_json_object(
|
||||
value: object,
|
||||
) -> TypeIs[Mapping[str, object]]: # guard-ok: cost-map rows and request params are JSON
|
||||
return isinstance(value, Mapping)
|
||||
|
||||
|
||||
def _json_field(value: object, key: str) -> object:
|
||||
return value.get(key) if _is_json_object(value) else None
|
||||
|
||||
|
||||
def _exceeds_output_limit(model: str, max_tokens: object) -> bool:
|
||||
info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai")
|
||||
limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens")
|
||||
return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit
|
||||
|
||||
|
||||
def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None:
|
||||
if _json_field(response_format, "type") != "json_schema":
|
||||
return None
|
||||
if litellm.supports_response_schema(model=model, custom_llm_provider="zai"):
|
||||
return None
|
||||
schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema")
|
||||
return schema if _is_json_object(schema) else None
|
||||
|
||||
|
||||
class ZAIChatConfig(OpenAIGPTConfig):
|
||||
|
|
@ -72,11 +102,39 @@ class ZAIChatConfig(OpenAIGPTConfig):
|
|||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params}
|
||||
max_tokens: Final = sent.get("max_completion_tokens", sent.get("max_tokens"))
|
||||
drops_max_tokens: Final = max_tokens is None or (
|
||||
(litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_tokens)
|
||||
)
|
||||
return {
|
||||
**optional_params,
|
||||
**{
|
||||
("max_tokens" if param == "max_completion_tokens" else param): value
|
||||
for param, value in non_default_params.items()
|
||||
if param in supported_params
|
||||
},
|
||||
**{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS},
|
||||
**({} if drops_max_tokens else {"max_tokens": max_tokens}),
|
||||
}
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature
|
||||
optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
|
||||
litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature
|
||||
headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature
|
||||
) -> dict[str, object]: # mutable-ok: inherited contract returns the request body
|
||||
schema: Final = _schema_without_native_support(model, optional_params.get("response_format"))
|
||||
schema_message: Final = (
|
||||
()
|
||||
if schema is None
|
||||
else (
|
||||
ChatCompletionUserMessage(
|
||||
role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai")
|
||||
),
|
||||
)
|
||||
)
|
||||
return super().transform_request(
|
||||
model,
|
||||
[*messages, *schema_message],
|
||||
optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}},
|
||||
litellm_params,
|
||||
headers,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -54308,6 +54308,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zai/glm-5.3-flash": {
|
||||
|
|
@ -54322,6 +54323,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://docs.z.ai/guides/overview/pricing",
|
||||
"supports_vision": true
|
||||
|
|
|
|||
|
|
@ -54308,6 +54308,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"zai/glm-5.3-flash": {
|
||||
|
|
@ -54322,6 +54323,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://docs.z.ai/guides/overview/pricing",
|
||||
"supports_vision": true
|
||||
|
|
|
|||
|
|
@ -10,11 +10,15 @@ import pytest
|
|||
import litellm
|
||||
from litellm import completion
|
||||
from litellm.cost_calculator import cost_per_token
|
||||
from litellm.llms.zai.chat.transformation import ZAIChatConfig
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
JSON_SCHEMA_RESPONSE_FORMAT = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"name": {"type": "string"}}}},
|
||||
"json_schema": {
|
||||
"name": "profile",
|
||||
"schema": {"type": "object", "properties": {"favorite_number": {"type": "integer"}}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -130,32 +134,72 @@ def test_zai_sends_response_format_and_maps_max_completion_tokens(respx_mock, za
|
|||
"token_limits",
|
||||
[{"max_tokens": 300, "max_completion_tokens": 15}, {"max_completion_tokens": 15, "max_tokens": 300}],
|
||||
)
|
||||
def test_zai_max_completion_tokens_wins_over_max_tokens(respx_mock, zai_response, monkeypatch, token_limits):
|
||||
def test_zai_max_completion_tokens_wins_over_max_tokens(token_limits):
|
||||
mapped = ZAIChatConfig().map_openai_params(token_limits, {}, "glm-5.3", drop_params=False)
|
||||
|
||||
assert mapped["max_tokens"] == 15
|
||||
assert "max_completion_tokens" not in mapped
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("over_limit_by", "drop_params", "forwarded"),
|
||||
[(1, True, False), (1, False, True), (0, True, True)],
|
||||
)
|
||||
def test_zai_drop_params_drops_max_completion_tokens_above_model_limit(
|
||||
local_model_cost_map, respx_mock, zai_response, monkeypatch, over_limit_by, drop_params, forwarded
|
||||
):
|
||||
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
|
||||
requested = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + over_limit_by
|
||||
|
||||
completion(model="zai/glm-5.3", messages=[{"role": "user", "content": "Count"}], **token_limits)
|
||||
completion(
|
||||
model="zai/glm-4.5-flash",
|
||||
messages=[{"role": "user", "content": "Count"}],
|
||||
max_completion_tokens=requested,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
body = json.loads(respx_mock.calls[0].request.content)
|
||||
assert body["max_tokens"] == 15
|
||||
assert body.get("max_tokens") == (requested if forwarded else None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}])
|
||||
def test_zai_forwards_json_schema_response_format_as_sent(respx_mock, zai_response, monkeypatch, pass_through):
|
||||
def test_zai_forwards_json_schema_to_models_that_enforce_it(
|
||||
local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through
|
||||
):
|
||||
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
|
||||
messages = [{"role": "user", "content": "Ann likes 7"}]
|
||||
|
||||
completion(model="zai/glm-5.3", messages=messages, response_format=JSON_SCHEMA_RESPONSE_FORMAT, **pass_through)
|
||||
|
||||
body = json.loads(respx_mock.calls[0].request.content)
|
||||
assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
|
||||
assert body["messages"] == messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}])
|
||||
def test_zai_puts_json_schema_in_prompt_for_models_that_ignore_it(
|
||||
local_model_cost_map, respx_mock, zai_response, monkeypatch, pass_through
|
||||
):
|
||||
monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response)
|
||||
|
||||
completion(
|
||||
model="zai/glm-5.3",
|
||||
messages=[{"role": "user", "content": "Ann is 30"}],
|
||||
model="zai/glm-4.6",
|
||||
messages=[{"role": "user", "content": "Ann likes 7"}],
|
||||
response_format=JSON_SCHEMA_RESPONSE_FORMAT,
|
||||
**pass_through,
|
||||
)
|
||||
|
||||
body = json.loads(respx_mock.calls[0].request.content)
|
||||
assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
|
||||
assert body["response_format"] == {"type": "json_object"}
|
||||
assert body["messages"][0] == {"role": "user", "content": "Ann likes 7"}
|
||||
assert body["messages"][-1]["role"] == "user"
|
||||
assert "favorite_number" in body["messages"][-1]["content"]
|
||||
|
||||
|
||||
def test_zai_glm_5v_turbo_bills_cached_input_at_cache_read_price(local_model_cost_map: None) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue