Merge 95e321cc75 into c2c2a623c0

litellm/llms/base_llm/chat/transformation.py

@@ -223,8 +223,12 @@ class BaseConfig(ABC):
                 function=ChatCompletionToolParamFunctionChunk(name=RESPONSE_FORMAT_TOOL_NAME, parameters=json_schema),
             )
 
-            optional_params.setdefault("tools", [])
-            optional_params["tools"].append(_tool)
+            # Rebind instead of appending: _map_openai_params aliases the caller's own
+            # tools list into optional_params, so append() would mutate it across calls.
+            optional_params["tools"] = [  # mutable-ok: JSON request body
+                *optional_params.get("tools", []),
+                _tool,
+            ]
             if enforce_tool_choice:
                 optional_params["tool_choice"] = _tool_choice
 

litellm/llms/openai/openai.py

@@ -673,6 +673,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             inference_params = optional_params.copy()
             stream_options: Final[dict | None] = inference_params.pop("stream_options", None)
             stream: Final[bool | None] = inference_params.pop("stream", False)
+            # Set by BaseConfig._add_response_format_to_tools, not an SDK argument. The
+            # OpenAI SDK rejects unknown kwargs, so it has to come off before the call.
+            json_mode: Final[bool | None] = inference_params.pop("json_mode", False)
             provider_config: BaseConfig | None = None
 
             if custom_llm_provider is not None and model is not None:
@@ -750,6 +753,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                                 drop_params=drop_params,
                                 fake_stream=fake_stream,
                                 shared_session=shared_session,
+                                json_mode=json_mode,
                             )
 
                     data = provider_config.transform_request(
@@ -824,6 +828,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                             response_object=stringified_response,
                             model_response_object=model_response,
                             _response_headers=headers,
+                            convert_tool_call_to_json_mode=json_mode,
                         )
                         if fake_stream is True:
                             return self.mock_streaming(
@@ -907,6 +912,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         stream_options: dict | None = None,
         fake_stream: bool = False,
         shared_session: Optional["ClientSession"] = None,
+        json_mode: bool | None = None,
     ):
         response = None
         data = await provider_config.async_transform_request(
@@ -962,6 +968,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     model_response_object=model_response,
                     hidden_params={"headers": headers},
                     _response_headers=headers,
+                    convert_tool_call_to_json_mode=json_mode,
                 )
 
                 # Call agentic completion hooks (e.g., for websearch_interception)

litellm/llms/zai/chat/transformation.py

@@ -43,6 +43,7 @@ class ZAIChatConfig(OpenAIGPTConfig):
             "stop",
             "tools",
             "tool_choice",
+            "response_format",
         ]
 
         import litellm
@@ -54,3 +55,61 @@ class ZAIChatConfig(OpenAIGPTConfig):
             pass
 
         return base_params
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        """
+        Translate response_format into a forced tool call.
+
+        GLM ignores response_format but honours a forced tool call, so a schema only
+        survives the round trip as one. See BerriAI/litellm#37720.
+        """
+        response_format: Final = non_default_params.get("response_format")
+        optional_params = super().map_openai_params(
+            non_default_params=non_default_params,
+            optional_params=optional_params,
+            model=model,
+            drop_params=drop_params,
+        )
+        if not self._should_translate_response_format(non_default_params, response_format):
+            return optional_params
+
+        # super() copied response_format through as an allowlisted param. GLM ignores
+        # it, so trade it for the tool call the model does honour.
+        optional_params.pop("response_format", None)
+        return self._add_response_format_to_tools(
+            optional_params=optional_params,
+            value=response_format,
+            is_response_format_supported=False,
+        )
+
+    @staticmethod
+    def _should_translate_response_format(non_default_params: dict, response_format: dict | None) -> bool:
+        """
+        Only translate when the forced tool call can actually be unwrapped again.
+
+        Otherwise response_format is left to pass through: GLM ignores it, which is
+        the pre-existing behaviour and better than an unusable tool call.
+        """
+        if response_format is None:
+            return False
+        # Forcing json_tool_call would hijack a caller that is genuinely using tools.
+        if non_default_params.get("tools"):
+            return False
+        # The unwrap back into message.content only runs on non-streaming responses.
+        if non_default_params.get("stream"):
+            return False
+        # Mirror _add_response_format_to_tools' own extraction, key presence included,
+        # so this never pops response_format for a schema the helper would ignore.
+        if "response_schema" in response_format:
+            json_schema = response_format["response_schema"]
+        elif "json_schema" in response_format:
+            json_schema = response_format["json_schema"].get("schema")
+        else:
+            json_schema = None
+        return bool(json_schema)

tests/test_litellm/llms/zai/test_zai_response_format.py

@@ -0,0 +1,274 @@
+"""
+Tests for response_format support on the Z.AI (GLM) provider.
+
+GLM ignores response_format but honours a forced tool call, so litellm translates a
+schema into one. Regression cover for BerriAI/litellm#37720.
+"""
+
+import json
+
+import httpx
+import pytest
+
+import litellm
+from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
+from litellm.llms.zai.chat.transformation import ZAIChatConfig
+from litellm.utils import get_optional_params
+
+ZAI_URL = "https://api.z.ai/api/paas/v4/chat/completions"
+
+SCHEMA = {
+    "type": "object",
+    "properties": {"entities": {"type": "array", "items": {"type": "string"}}},
+    "required": ["entities"],
+    "additionalProperties": False,
+}
+JSON_SCHEMA_FORMAT = {
+    "type": "json_schema",
+    "json_schema": {"name": "ents", "strict": True, "schema": SCHEMA},
+}
+TOOL_ARGUMENTS = json.dumps({"entities": ["Alice", "service"]})
+MESSAGES = [{"role": "user", "content": "Extract entities from: Alice deployed a service."}]
+
+
+@pytest.fixture
+def tool_call_reply():
+    """Z.AI reply to a forced json_tool_call: the schema-valid payload is the arguments."""
+    return {
+        "id": "chatcmpl-zai-rf",
+        "object": "chat.completion",
+        "created": 1677652288,
+        "model": "glm-5-turbo",
+        "choices": [
+            {
+                "index": 0,
+                "finish_reason": "tool_calls",
+                "message": {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": "call_1",
+                            "type": "function",
+                            "function": {
+                                "name": RESPONSE_FORMAT_TOOL_NAME,
+                                "arguments": TOOL_ARGUMENTS,
+                            },
+                        }
+                    ],
+                },
+            }
+        ],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+
+
+def test_response_format_is_a_supported_param():
+    """response_format must be allowlisted, or get_optional_params drops it before mapping."""
+    assert "response_format" in ZAIChatConfig().get_supported_openai_params(model="glm-5-turbo")
+
+
+def test_json_schema_becomes_a_forced_tool_call():
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format=JSON_SCHEMA_FORMAT,
+    )
+
+    assert "response_format" not in optional_params, "GLM ignores response_format; it must be translated"
+    assert optional_params["tools"] == [
+        {
+            "type": "function",
+            "function": {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": SCHEMA},
+        }
+    ]
+    assert optional_params["tool_choice"] == {
+        "type": "function",
+        "function": {"name": RESPONSE_FORMAT_TOOL_NAME},
+    }
+    assert optional_params["json_mode"] is True
+
+
+def test_json_schema_is_translated_even_when_drop_params_is_set():
+    """The silent-drop path in the bug report: drop_params must no longer swallow the schema."""
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format=JSON_SCHEMA_FORMAT,
+        drop_params=True,
+    )
+
+    assert optional_params["tools"][0]["function"]["parameters"] == SCHEMA
+    assert optional_params["json_mode"] is True
+
+
+def test_json_object_is_forwarded_unchanged():
+    """No schema to translate. GLM returns arbitrary-shaped but valid JSON, so forward it."""
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format={"type": "json_object"},
+    )
+
+    assert optional_params["response_format"] == {"type": "json_object"}
+    assert "json_mode" not in optional_params
+    assert "tools" not in optional_params
+
+
+def _caller_tool():
+    return {
+        "type": "function",
+        "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
+    }
+
+
+def test_caller_supplied_tools_are_left_alone():
+    """
+    Forcing json_tool_call would hijack a caller that is genuinely using tools, and an
+    unforced one would leave nothing constraining the model. Neither is translated.
+    """
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        tools=[_caller_tool()],
+        response_format=JSON_SCHEMA_FORMAT,
+    )
+
+    assert optional_params["tools"] == [_caller_tool()]
+    assert "tool_choice" not in optional_params
+    assert optional_params["response_format"] == JSON_SCHEMA_FORMAT
+    assert "json_mode" not in optional_params
+
+
+def test_caller_tools_list_is_not_mutated_across_calls():
+    """
+    _map_openai_params aliases the caller's list into optional_params, so appending the
+    generated tool would corrupt a module-level TOOLS list on every turn of an agent loop.
+    """
+    caller_tools = [_caller_tool()]
+
+    for _ in range(3):
+        get_optional_params(
+            model="glm-5-turbo",
+            custom_llm_provider="zai",
+            tools=caller_tools,
+            response_format=JSON_SCHEMA_FORMAT,
+        )
+
+    assert [tool["function"]["name"] for tool in caller_tools] == ["get_weather"]
+
+
+def test_base_helper_does_not_mutate_the_tools_it_is_given():
+    """Direct cover for the shared helper, which azure and fireworks_ai also call."""
+    caller_tools = [_caller_tool()]
+    optional_params = ZAIChatConfig()._add_response_format_to_tools(
+        optional_params={"tools": caller_tools},
+        value=JSON_SCHEMA_FORMAT,
+        is_response_format_supported=False,
+    )
+
+    assert len(optional_params["tools"]) == 2
+    assert [tool["function"]["name"] for tool in caller_tools] == ["get_weather"]
+
+
+def test_falsy_response_schema_is_not_silently_dropped():
+    """
+    The helper extracts response_schema by key presence, so a present-but-empty one makes
+    it a no-op. Translating anyway would pop response_format and lose the schema entirely.
+    """
+    response_format = {
+        "type": "json_schema",
+        "response_schema": {},
+        "json_schema": {"name": "ents", "schema": SCHEMA},
+    }
+
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format=response_format,
+    )
+
+    assert optional_params["response_format"] == response_format
+    assert "tools" not in optional_params
+
+
+def test_completion_unwraps_the_tool_call_into_content(respx_mock, tool_call_reply, monkeypatch):
+    """End-to-end: json_mode must not reach the SDK, and the arguments become the content."""
+    monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route = respx_mock.post(ZAI_URL).mock(return_value=httpx.Response(200, json=tool_call_reply))
+
+    response = litellm.completion(
+        model="zai/glm-5-turbo",
+        messages=MESSAGES,
+        response_format=JSON_SCHEMA_FORMAT,
+        max_tokens=100,
+    )
+
+    assert json.loads(response.choices[0].message.content) == {"entities": ["Alice", "service"]}
+    assert response.choices[0].message.tool_calls is None
+    assert response.choices[0].finish_reason == "stop"
+
+    request_body = json.loads(route.calls[0].request.content)
+    assert "json_mode" not in request_body, "json_mode is internal; the OpenAI SDK rejects it"
+    assert request_body["tool_choice"]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME
+
+
+@pytest.mark.asyncio
+async def test_acompletion_unwraps_the_tool_call_into_content(respx_mock, tool_call_reply, monkeypatch):
+    monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route = respx_mock.post(ZAI_URL).mock(return_value=httpx.Response(200, json=tool_call_reply))
+
+    response = await litellm.acompletion(
+        model="zai/glm-5-turbo",
+        messages=MESSAGES,
+        response_format=JSON_SCHEMA_FORMAT,
+        max_tokens=100,
+    )
+
+    assert json.loads(response.choices[0].message.content) == {"entities": ["Alice", "service"]}
+    assert response.choices[0].finish_reason == "stop"
+
+    request_body = json.loads(route.calls[0].request.content)
+    assert "json_mode" not in request_body
+
+
+def test_streaming_is_left_untranslated(respx_mock, monkeypatch):
+    """
+    The unwrap back into message.content only runs on non-streaming responses, so a
+    forced tool call here would hand the caller a stream of unusable tool_calls.
+    """
+    monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    chunk = {
+        "id": "chatcmpl-zai-rf",
+        "object": "chat.completion.chunk",
+        "created": 1677652288,
+        "model": "glm-5-turbo",
+        "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+    }
+    route = respx_mock.post(ZAI_URL).mock(
+        return_value=httpx.Response(
+            200,
+            text=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n",
+            headers={"content-type": "text/event-stream"},
+        )
+    )
+
+    chunks = list(
+        litellm.completion(
+            model="zai/glm-5-turbo",
+            messages=MESSAGES,
+            response_format=JSON_SCHEMA_FORMAT,
+            max_tokens=100,
+            stream=True,
+        )
+    )
+
+    assert [chunk.choices[0].delta.content for chunk in chunks][0] == "hi"
+    request_body = json.loads(route.calls[0].request.content)
+    assert "json_mode" not in request_body
+    assert "tool_choice" not in request_body, "a forced tool call cannot be unwrapped mid-stream"
+    assert request_body["response_format"] == JSON_SCHEMA_FORMAT
+    assert request_body["stream"] is True