diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index d147063df73..2d4620bbd05 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -219,8 +219,12 @@ class BaseConfig(ABC): function=ChatCompletionToolParamFunctionChunk(name=RESPONSE_FORMAT_TOOL_NAME, parameters=json_schema), ) - optional_params.setdefault("tools", []) - optional_params["tools"].append(_tool) + # Rebind instead of appending: _map_openai_params aliases the caller's own + # tools list into optional_params, so append() would mutate it across calls. + optional_params["tools"] = [ # mutable-ok: JSON request body + *optional_params.get("tools", []), + _tool, + ] if enforce_tool_choice: optional_params["tool_choice"] = _tool_choice diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 4fc6655ca54..e47572bc627 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -616,6 +616,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): inference_params = optional_params.copy() stream_options: Final[dict | None] = inference_params.pop("stream_options", None) stream: Final[bool | None] = inference_params.pop("stream", False) + # Set by BaseConfig._add_response_format_to_tools, not an SDK argument. The + # OpenAI SDK rejects unknown kwargs, so it has to come off before the call. + json_mode: Final[bool | None] = inference_params.pop("json_mode", False) provider_config: BaseConfig | None = None if custom_llm_provider is not None and model is not None: @@ -693,6 +696,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): drop_params=drop_params, fake_stream=fake_stream, shared_session=shared_session, + json_mode=json_mode, ) data = provider_config.transform_request( @@ -766,6 +770,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): response_object=stringified_response, model_response_object=model_response, _response_headers=headers, + convert_tool_call_to_json_mode=json_mode, ) if fake_stream is True: return self.mock_streaming( @@ -849,6 +854,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): stream_options: dict | None = None, fake_stream: bool = False, shared_session: Optional["ClientSession"] = None, + json_mode: bool | None = None, ): response = None data = await provider_config.async_transform_request( @@ -903,6 +909,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): model_response_object=model_response, hidden_params={"headers": headers}, _response_headers=headers, + convert_tool_call_to_json_mode=json_mode, ) # Call agentic completion hooks (e.g., for websearch_interception) diff --git a/litellm/llms/zai/chat/transformation.py b/litellm/llms/zai/chat/transformation.py index b53eef19d1e..2e2682c29b8 100644 --- a/litellm/llms/zai/chat/transformation.py +++ b/litellm/llms/zai/chat/transformation.py @@ -43,6 +43,7 @@ class ZAIChatConfig(OpenAIGPTConfig): "stop", "tools", "tool_choice", + "response_format", ] import litellm @@ -54,3 +55,61 @@ class ZAIChatConfig(OpenAIGPTConfig): pass return base_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Translate response_format into a forced tool call. + + GLM ignores response_format but honours a forced tool call, so a schema only + survives the round trip as one. See BerriAI/litellm#37720. + """ + response_format: Final = non_default_params.get("response_format") + optional_params = super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + if not self._should_translate_response_format(non_default_params, response_format): + return optional_params + + # super() copied response_format through as an allowlisted param. GLM ignores + # it, so trade it for the tool call the model does honour. + optional_params.pop("response_format", None) + return self._add_response_format_to_tools( + optional_params=optional_params, + value=response_format, + is_response_format_supported=False, + ) + + @staticmethod + def _should_translate_response_format(non_default_params: dict, response_format: dict | None) -> bool: + """ + Only translate when the forced tool call can actually be unwrapped again. + + Otherwise response_format is left to pass through: GLM ignores it, which is + the pre-existing behaviour and better than an unusable tool call. + """ + if response_format is None: + return False + # Forcing json_tool_call would hijack a caller that is genuinely using tools. + if non_default_params.get("tools"): + return False + # The unwrap back into message.content only runs on non-streaming responses. + if non_default_params.get("stream"): + return False + # Mirror _add_response_format_to_tools' own extraction, key presence included, + # so this never pops response_format for a schema the helper would ignore. + if "response_schema" in response_format: + json_schema = response_format["response_schema"] + elif "json_schema" in response_format: + json_schema = response_format["json_schema"].get("schema") + else: + json_schema = None + return bool(json_schema) diff --git a/tests/test_litellm/llms/zai/test_zai_response_format.py b/tests/test_litellm/llms/zai/test_zai_response_format.py new file mode 100644 index 00000000000..688f5fcda47 --- /dev/null +++ b/tests/test_litellm/llms/zai/test_zai_response_format.py @@ -0,0 +1,274 @@ +""" +Tests for response_format support on the Z.AI (GLM) provider. + +GLM ignores response_format but honours a forced tool call, so litellm translates a +schema into one. Regression cover for BerriAI/litellm#37720. +""" + +import json + +import httpx +import pytest + +import litellm +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.llms.zai.chat.transformation import ZAIChatConfig +from litellm.utils import get_optional_params + +ZAI_URL = "https://api.z.ai/api/paas/v4/chat/completions" + +SCHEMA = { + "type": "object", + "properties": {"entities": {"type": "array", "items": {"type": "string"}}}, + "required": ["entities"], + "additionalProperties": False, +} +JSON_SCHEMA_FORMAT = { + "type": "json_schema", + "json_schema": {"name": "ents", "strict": True, "schema": SCHEMA}, +} +TOOL_ARGUMENTS = json.dumps({"entities": ["Alice", "service"]}) +MESSAGES = [{"role": "user", "content": "Extract entities from: Alice deployed a service."}] + + +@pytest.fixture +def tool_call_reply(): + """Z.AI reply to a forced json_tool_call: the schema-valid payload is the arguments.""" + return { + "id": "chatcmpl-zai-rf", + "object": "chat.completion", + "created": 1677652288, + "model": "glm-5-turbo", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": RESPONSE_FORMAT_TOOL_NAME, + "arguments": TOOL_ARGUMENTS, + }, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + +def test_response_format_is_a_supported_param(): + """response_format must be allowlisted, or get_optional_params drops it before mapping.""" + assert "response_format" in ZAIChatConfig().get_supported_openai_params(model="glm-5-turbo") + + +def test_json_schema_becomes_a_forced_tool_call(): + optional_params = get_optional_params( + model="glm-5-turbo", + custom_llm_provider="zai", + response_format=JSON_SCHEMA_FORMAT, + ) + + assert "response_format" not in optional_params, "GLM ignores response_format; it must be translated" + assert optional_params["tools"] == [ + { + "type": "function", + "function": {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": SCHEMA}, + } + ] + assert optional_params["tool_choice"] == { + "type": "function", + "function": {"name": RESPONSE_FORMAT_TOOL_NAME}, + } + assert optional_params["json_mode"] is True + + +def test_json_schema_is_translated_even_when_drop_params_is_set(): + """The silent-drop path in the bug report: drop_params must no longer swallow the schema.""" + optional_params = get_optional_params( + model="glm-5-turbo", + custom_llm_provider="zai", + response_format=JSON_SCHEMA_FORMAT, + drop_params=True, + ) + + assert optional_params["tools"][0]["function"]["parameters"] == SCHEMA + assert optional_params["json_mode"] is True + + +def test_json_object_is_forwarded_unchanged(): + """No schema to translate. GLM returns arbitrary-shaped but valid JSON, so forward it.""" + optional_params = get_optional_params( + model="glm-5-turbo", + custom_llm_provider="zai", + response_format={"type": "json_object"}, + ) + + assert optional_params["response_format"] == {"type": "json_object"} + assert "json_mode" not in optional_params + assert "tools" not in optional_params + + +def _caller_tool(): + return { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, + } + + +def test_caller_supplied_tools_are_left_alone(): + """ + Forcing json_tool_call would hijack a caller that is genuinely using tools, and an + unforced one would leave nothing constraining the model. Neither is translated. + """ + optional_params = get_optional_params( + model="glm-5-turbo", + custom_llm_provider="zai", + tools=[_caller_tool()], + response_format=JSON_SCHEMA_FORMAT, + ) + + assert optional_params["tools"] == [_caller_tool()] + assert "tool_choice" not in optional_params + assert optional_params["response_format"] == JSON_SCHEMA_FORMAT + assert "json_mode" not in optional_params + + +def test_caller_tools_list_is_not_mutated_across_calls(): + """ + _map_openai_params aliases the caller's list into optional_params, so appending the + generated tool would corrupt a module-level TOOLS list on every turn of an agent loop. + """ + caller_tools = [_caller_tool()] + + for _ in range(3): + get_optional_params( + model="glm-5-turbo", + custom_llm_provider="zai", + tools=caller_tools, + response_format=JSON_SCHEMA_FORMAT, + ) + + assert [tool["function"]["name"] for tool in caller_tools] == ["get_weather"] + + +def test_base_helper_does_not_mutate_the_tools_it_is_given(): + """Direct cover for the shared helper, which azure and fireworks_ai also call.""" + caller_tools = [_caller_tool()] + optional_params = ZAIChatConfig()._add_response_format_to_tools( + optional_params={"tools": caller_tools}, + value=JSON_SCHEMA_FORMAT, + is_response_format_supported=False, + ) + + assert len(optional_params["tools"]) == 2 + assert [tool["function"]["name"] for tool in caller_tools] == ["get_weather"] + + +def test_falsy_response_schema_is_not_silently_dropped(): + """ + The helper extracts response_schema by key presence, so a present-but-empty one makes + it a no-op. Translating anyway would pop response_format and lose the schema entirely. + """ + response_format = { + "type": "json_schema", + "response_schema": {}, + "json_schema": {"name": "ents", "schema": SCHEMA}, + } + + optional_params = get_optional_params( + model="glm-5-turbo", + custom_llm_provider="zai", + response_format=response_format, + ) + + assert optional_params["response_format"] == response_format + assert "tools" not in optional_params + + +def test_completion_unwraps_the_tool_call_into_content(respx_mock, tool_call_reply, monkeypatch): + """End-to-end: json_mode must not reach the SDK, and the arguments become the content.""" + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(ZAI_URL).mock(return_value=httpx.Response(200, json=tool_call_reply)) + + response = litellm.completion( + model="zai/glm-5-turbo", + messages=MESSAGES, + response_format=JSON_SCHEMA_FORMAT, + max_tokens=100, + ) + + assert json.loads(response.choices[0].message.content) == {"entities": ["Alice", "service"]} + assert response.choices[0].message.tool_calls is None + assert response.choices[0].finish_reason == "stop" + + request_body = json.loads(route.calls[0].request.content) + assert "json_mode" not in request_body, "json_mode is internal; the OpenAI SDK rejects it" + assert request_body["tool_choice"]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME + + +@pytest.mark.asyncio +async def test_acompletion_unwraps_the_tool_call_into_content(respx_mock, tool_call_reply, monkeypatch): + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(ZAI_URL).mock(return_value=httpx.Response(200, json=tool_call_reply)) + + response = await litellm.acompletion( + model="zai/glm-5-turbo", + messages=MESSAGES, + response_format=JSON_SCHEMA_FORMAT, + max_tokens=100, + ) + + assert json.loads(response.choices[0].message.content) == {"entities": ["Alice", "service"]} + assert response.choices[0].finish_reason == "stop" + + request_body = json.loads(route.calls[0].request.content) + assert "json_mode" not in request_body + + +def test_streaming_is_left_untranslated(respx_mock, monkeypatch): + """ + The unwrap back into message.content only runs on non-streaming responses, so a + forced tool call here would hand the caller a stream of unusable tool_calls. + """ + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chunk = { + "id": "chatcmpl-zai-rf", + "object": "chat.completion.chunk", + "created": 1677652288, + "model": "glm-5-turbo", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + } + route = respx_mock.post(ZAI_URL).mock( + return_value=httpx.Response( + 200, + text=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n", + headers={"content-type": "text/event-stream"}, + ) + ) + + chunks = list( + litellm.completion( + model="zai/glm-5-turbo", + messages=MESSAGES, + response_format=JSON_SCHEMA_FORMAT, + max_tokens=100, + stream=True, + ) + ) + + assert [chunk.choices[0].delta.content for chunk in chunks][0] == "hi" + request_body = json.loads(route.calls[0].request.content) + assert "json_mode" not in request_body + assert "tool_choice" not in request_body, "a forced tool call cannot be unwrapped mid-stream" + assert request_body["response_format"] == JSON_SCHEMA_FORMAT + assert request_body["stream"] is True