fix(zai): translate response_format into a forced tool call

ZAIChatConfig.get_supported_openai_params omitted response_format, so litellm either raised
UnsupportedParamsError or, under drop_params, dropped the schema and returned HTTP 200 with prose.
The silent variant is the damaging one: the failure only surfaces in a downstream JSON parser.

GLM ignores response_format upstream but does honour a forced tool call, which is what
BaseConfig._add_response_format_to_tools already builds for azure and fireworks_ai. Wiring zai into
it is not enough on its own: that helper sets the internal json_mode flag, and zai reaches the
OpenAI SDK handler, which passed the flag straight through to Completions.create() as an unexpected
keyword argument. azure solved the same problem by popping json_mode and threading
convert_tool_call_to_json_mode into its response conversion, so openai.py now does the same and the
tool call is unwrapped back into message.content.

The translation is skipped when it could not be unwrapped again: streaming responses, and requests
that already carry caller-supplied tools, where forcing json_tool_call would hijack them. In both
cases response_format is left to pass through as before.

Also stops _add_response_format_to_tools mutating the caller's tools list. _map_openai_params
aliases that list into optional_params, so the previous setdefault plus append wrote into the list
the caller passed to completion(), accumulating a duplicate json_tool_call on every turn of an
agent loop. azure and fireworks_ai were exposed to this too.

Fixes: BerriAI/litellm#37720
Signed-off-by: Alexander Chernov <alexander@chernov.it>

litellm/llms/base_llm/chat/transformation.py

@@ -219,8 +219,12 @@ class BaseConfig(ABC):
                 function=ChatCompletionToolParamFunctionChunk(name=RESPONSE_FORMAT_TOOL_NAME, parameters=json_schema),
             )
 
-            optional_params.setdefault("tools", [])
-            optional_params["tools"].append(_tool)
+            # Rebind instead of appending: _map_openai_params aliases the caller's own
+            # tools list into optional_params, so append() would mutate it across calls.
+            optional_params["tools"] = [  # mutable-ok: JSON request body
+                *optional_params.get("tools", []),
+                _tool,
+            ]
             if enforce_tool_choice:
                 optional_params["tool_choice"] = _tool_choice
 

litellm/llms/openai/openai.py

@@ -616,6 +616,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             inference_params = optional_params.copy()
             stream_options: Final[dict | None] = inference_params.pop("stream_options", None)
             stream: Final[bool | None] = inference_params.pop("stream", False)
+            # Set by BaseConfig._add_response_format_to_tools, not an SDK argument. The
+            # OpenAI SDK rejects unknown kwargs, so it has to come off before the call.
+            json_mode: Final[bool | None] = inference_params.pop("json_mode", False)
             provider_config: BaseConfig | None = None
 
             if custom_llm_provider is not None and model is not None:
@@ -693,6 +696,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                                 drop_params=drop_params,
                                 fake_stream=fake_stream,
                                 shared_session=shared_session,
+                                json_mode=json_mode,
                             )
 
                     data = provider_config.transform_request(
@@ -766,6 +770,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                             response_object=stringified_response,
                             model_response_object=model_response,
                             _response_headers=headers,
+                            convert_tool_call_to_json_mode=json_mode,
                         )
                         if fake_stream is True:
                             return self.mock_streaming(
@@ -849,6 +854,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         stream_options: dict | None = None,
         fake_stream: bool = False,
         shared_session: Optional["ClientSession"] = None,
+        json_mode: bool | None = None,
     ):
         response = None
         data = await provider_config.async_transform_request(
@@ -903,6 +909,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     model_response_object=model_response,
                     hidden_params={"headers": headers},
                     _response_headers=headers,
+                    convert_tool_call_to_json_mode=json_mode,
                 )
 
                 # Call agentic completion hooks (e.g., for websearch_interception)

litellm/llms/zai/chat/transformation.py

@@ -43,6 +43,7 @@ class ZAIChatConfig(OpenAIGPTConfig):
             "stop",
             "tools",
             "tool_choice",
+            "response_format",
         ]
 
         import litellm
@@ -54,3 +55,61 @@ class ZAIChatConfig(OpenAIGPTConfig):
             pass
 
         return base_params
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        """
+        Translate response_format into a forced tool call.
+
+        GLM ignores response_format but honours a forced tool call, so a schema only
+        survives the round trip as one. See BerriAI/litellm#37720.
+        """
+        response_format: Final = non_default_params.get("response_format")
+        optional_params = super().map_openai_params(
+            non_default_params=non_default_params,
+            optional_params=optional_params,
+            model=model,
+            drop_params=drop_params,
+        )
+        if not self._should_translate_response_format(non_default_params, response_format):
+            return optional_params
+
+        # super() copied response_format through as an allowlisted param. GLM ignores
+        # it, so trade it for the tool call the model does honour.
+        optional_params.pop("response_format", None)
+        return self._add_response_format_to_tools(
+            optional_params=optional_params,
+            value=response_format,
+            is_response_format_supported=False,
+        )
+
+    @staticmethod
+    def _should_translate_response_format(non_default_params: dict, response_format: dict | None) -> bool:
+        """
+        Only translate when the forced tool call can actually be unwrapped again.
+
+        Otherwise response_format is left to pass through: GLM ignores it, which is
+        the pre-existing behaviour and better than an unusable tool call.
+        """
+        if response_format is None:
+            return False
+        # Forcing json_tool_call would hijack a caller that is genuinely using tools.
+        if non_default_params.get("tools"):
+            return False
+        # The unwrap back into message.content only runs on non-streaming responses.
+        if non_default_params.get("stream"):
+            return False
+        # Mirror _add_response_format_to_tools' own extraction, key presence included,
+        # so this never pops response_format for a schema the helper would ignore.
+        if "response_schema" in response_format:
+            json_schema = response_format["response_schema"]
+        elif "json_schema" in response_format:
+            json_schema = response_format["json_schema"].get("schema")
+        else:
+            json_schema = None
+        return bool(json_schema)

tests/test_litellm/llms/zai/test_zai_response_format.py

@@ -0,0 +1,274 @@
+"""
+Tests for response_format support on the Z.AI (GLM) provider.
+
+GLM ignores response_format but honours a forced tool call, so litellm translates a
+schema into one. Regression cover for BerriAI/litellm#37720.
+"""
+
+import json
+
+import httpx
+import pytest
+
+import litellm
+from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
+from litellm.llms.zai.chat.transformation import ZAIChatConfig
+from litellm.utils import get_optional_params
+
+ZAI_URL = "https://api.z.ai/api/paas/v4/chat/completions"
+
+SCHEMA = {
+    "type": "object",
+    "properties": {"entities": {"type": "array", "items": {"type": "string"}}},
+    "required": ["entities"],
+    "additionalProperties": False,
+}
+JSON_SCHEMA_FORMAT = {
+    "type": "json_schema",
+    "json_schema": {"name": "ents", "strict": True, "schema": SCHEMA},
+}
+TOOL_ARGUMENTS = json.dumps({"entities": ["Alice", "service"]})
+MESSAGES = [{"role": "user", "content": "Extract entities from: Alice deployed a service."}]
+
+
+@pytest.fixture
+def tool_call_reply():
+    """Z.AI reply to a forced json_tool_call: the schema-valid payload is the arguments."""
+    return {
+        "id": "chatcmpl-zai-rf",
+        "object": "chat.completion",
+        "created": 1677652288,
+        "model": "glm-5-turbo",
+        "choices": [
+            {
+                "index": 0,
+                "finish_reason": "tool_calls",
+                "message": {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": "call_1",
+                            "type": "function",
+                            "function": {
+                                "name": RESPONSE_FORMAT_TOOL_NAME,
+                                "arguments": TOOL_ARGUMENTS,
+                            },
+                        }
+                    ],
+                },
+            }
+        ],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+
+
+def test_response_format_is_a_supported_param():
+    """response_format must be allowlisted, or get_optional_params drops it before mapping."""
+    assert "response_format" in ZAIChatConfig().get_supported_openai_params(model="glm-5-turbo")
+
+
+def test_json_schema_becomes_a_forced_tool_call():
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format=JSON_SCHEMA_FORMAT,
+    )
+
+    assert "response_format" not in optional_params, "GLM ignores response_format; it must be translated"
+    assert optional_params["tools"] == [
+        {
+            "type": "function",
+            "function": {"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": SCHEMA},
+        }
+    ]
+    assert optional_params["tool_choice"] == {
+        "type": "function",
+        "function": {"name": RESPONSE_FORMAT_TOOL_NAME},
+    }
+    assert optional_params["json_mode"] is True
+
+
+def test_json_schema_is_translated_even_when_drop_params_is_set():
+    """The silent-drop path in the bug report: drop_params must no longer swallow the schema."""
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format=JSON_SCHEMA_FORMAT,
+        drop_params=True,
+    )
+
+    assert optional_params["tools"][0]["function"]["parameters"] == SCHEMA
+    assert optional_params["json_mode"] is True
+
+
+def test_json_object_is_forwarded_unchanged():
+    """No schema to translate. GLM returns arbitrary-shaped but valid JSON, so forward it."""
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format={"type": "json_object"},
+    )
+
+    assert optional_params["response_format"] == {"type": "json_object"}
+    assert "json_mode" not in optional_params
+    assert "tools" not in optional_params
+
+
+def _caller_tool():
+    return {
+        "type": "function",
+        "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
+    }
+
+
+def test_caller_supplied_tools_are_left_alone():
+    """
+    Forcing json_tool_call would hijack a caller that is genuinely using tools, and an
+    unforced one would leave nothing constraining the model. Neither is translated.
+    """
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        tools=[_caller_tool()],
+        response_format=JSON_SCHEMA_FORMAT,
+    )
+
+    assert optional_params["tools"] == [_caller_tool()]
+    assert "tool_choice" not in optional_params
+    assert optional_params["response_format"] == JSON_SCHEMA_FORMAT
+    assert "json_mode" not in optional_params
+
+
+def test_caller_tools_list_is_not_mutated_across_calls():
+    """
+    _map_openai_params aliases the caller's list into optional_params, so appending the
+    generated tool would corrupt a module-level TOOLS list on every turn of an agent loop.
+    """
+    caller_tools = [_caller_tool()]
+
+    for _ in range(3):
+        get_optional_params(
+            model="glm-5-turbo",
+            custom_llm_provider="zai",
+            tools=caller_tools,
+            response_format=JSON_SCHEMA_FORMAT,
+        )
+
+    assert [tool["function"]["name"] for tool in caller_tools] == ["get_weather"]
+
+
+def test_base_helper_does_not_mutate_the_tools_it_is_given():
+    """Direct cover for the shared helper, which azure and fireworks_ai also call."""
+    caller_tools = [_caller_tool()]
+    optional_params = ZAIChatConfig()._add_response_format_to_tools(
+        optional_params={"tools": caller_tools},
+        value=JSON_SCHEMA_FORMAT,
+        is_response_format_supported=False,
+    )
+
+    assert len(optional_params["tools"]) == 2
+    assert [tool["function"]["name"] for tool in caller_tools] == ["get_weather"]
+
+
+def test_falsy_response_schema_is_not_silently_dropped():
+    """
+    The helper extracts response_schema by key presence, so a present-but-empty one makes
+    it a no-op. Translating anyway would pop response_format and lose the schema entirely.
+    """
+    response_format = {
+        "type": "json_schema",
+        "response_schema": {},
+        "json_schema": {"name": "ents", "schema": SCHEMA},
+    }
+
+    optional_params = get_optional_params(
+        model="glm-5-turbo",
+        custom_llm_provider="zai",
+        response_format=response_format,
+    )
+
+    assert optional_params["response_format"] == response_format
+    assert "tools" not in optional_params
+
+
+def test_completion_unwraps_the_tool_call_into_content(respx_mock, tool_call_reply, monkeypatch):
+    """End-to-end: json_mode must not reach the SDK, and the arguments become the content."""
+    monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route = respx_mock.post(ZAI_URL).mock(return_value=httpx.Response(200, json=tool_call_reply))
+
+    response = litellm.completion(
+        model="zai/glm-5-turbo",
+        messages=MESSAGES,
+        response_format=JSON_SCHEMA_FORMAT,
+        max_tokens=100,
+    )
+
+    assert json.loads(response.choices[0].message.content) == {"entities": ["Alice", "service"]}
+    assert response.choices[0].message.tool_calls is None
+    assert response.choices[0].finish_reason == "stop"
+
+    request_body = json.loads(route.calls[0].request.content)
+    assert "json_mode" not in request_body, "json_mode is internal; the OpenAI SDK rejects it"
+    assert request_body["tool_choice"]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME
+
+
+@pytest.mark.asyncio
+async def test_acompletion_unwraps_the_tool_call_into_content(respx_mock, tool_call_reply, monkeypatch):
+    monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    route = respx_mock.post(ZAI_URL).mock(return_value=httpx.Response(200, json=tool_call_reply))
+
+    response = await litellm.acompletion(
+        model="zai/glm-5-turbo",
+        messages=MESSAGES,
+        response_format=JSON_SCHEMA_FORMAT,
+        max_tokens=100,
+    )
+
+    assert json.loads(response.choices[0].message.content) == {"entities": ["Alice", "service"]}
+    assert response.choices[0].finish_reason == "stop"
+
+    request_body = json.loads(route.calls[0].request.content)
+    assert "json_mode" not in request_body
+
+
+def test_streaming_is_left_untranslated(respx_mock, monkeypatch):
+    """
+    The unwrap back into message.content only runs on non-streaming responses, so a
+    forced tool call here would hand the caller a stream of unusable tool_calls.
+    """
+    monkeypatch.setenv("ZAI_API_KEY", "test-api-key")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    chunk = {
+        "id": "chatcmpl-zai-rf",
+        "object": "chat.completion.chunk",
+        "created": 1677652288,
+        "model": "glm-5-turbo",
+        "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
+    }
+    route = respx_mock.post(ZAI_URL).mock(
+        return_value=httpx.Response(
+            200,
+            text=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n",
+            headers={"content-type": "text/event-stream"},
+        )
+    )
+
+    chunks = list(
+        litellm.completion(
+            model="zai/glm-5-turbo",
+            messages=MESSAGES,
+            response_format=JSON_SCHEMA_FORMAT,
+            max_tokens=100,
+            stream=True,
+        )
+    )
+
+    assert [chunk.choices[0].delta.content for chunk in chunks][0] == "hi"
+    request_body = json.loads(route.calls[0].request.content)
+    assert "json_mode" not in request_body
+    assert "tool_choice" not in request_body, "a forced tool call cannot be unwrapped mid-stream"
+    assert request_body["response_format"] == JSON_SCHEMA_FORMAT
+    assert request_body["stream"] is True