fix(messages): honor OpenAI stop_sequences (#37118) (#48534)

litellm/llms/anthropic/pass_through/messages/handler.py

@@ -99,6 +99,7 @@ def _should_route_to_responses_api(
     requested_model: str | None = None,
     resolved_model: str | None = None,
     deployment_model_info: object = None,
+    stop_sequences: Sequence[str] | None = None,
 ) -> bool:
     """Return True when the request should use the Responses API path.
 
@@ -107,6 +108,8 @@ def _should_route_to_responses_api(
     """
     if litellm.use_chat_completions_url_for_anthropic_messages:
         return False
+    if stop_sequences:
+        return False
     if custom_llm_provider in _RESPONSES_API_PROVIDERS:
         return True
     if custom_llm_provider is None or requested_model is None or resolved_model is None:
@@ -690,7 +693,7 @@ def anthropic_messages_handler(
     if anthropic_messages_provider_config is None:
         # Route to Responses API for OpenAI / Azure, chat/completions for everything else.
         if kwargs.get("compaction") is None and _should_route_to_responses_api(
-            custom_llm_provider, original_model, model, kwargs.get("model_info")
+            custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences
         ):
             return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
                 max_tokens=max_tokens,

tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py

@@ -32,7 +32,58 @@ from litellm.utils import _invalidate_model_cost_lowercase_map
 import unittest.mock
 
 
-def test_anthropic_experimental_pass_through_messages_handler():
+@pytest.mark.asyncio
+@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
+@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"])
+async def test_openai_messages_forwards_stop_sequences_to_chat_completions(
+    monkeypatch: pytest.MonkeyPatch, is_async: bool, stream: bool
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    response: Final = (
+        httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-stop-sequences",
+                "object": "chat.completion",
+                "created": 1,
+                "model": "gpt-4o-mini",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1},
+            },
+        )
+        if not stream
+        else httpx.Response(
+            200,
+            headers={"content-type": "text/event-stream"},
+            text='data: {"id":"chatcmpl-stop-sequences","object":"chat.completion.chunk","created":1,'
+            '"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":""},"finish_reason":"stop"}]}\n\n'
+            "data: [DONE]\n\n",
+        )
+    )
+    with respx.mock as upstream:
+        chat: Final = upstream.post("https://api.openai.com/v1/chat/completions").mock(return_value=response)
+        create: Final = litellm.anthropic.messages.acreate if is_async else litellm.anthropic.messages.create
+        pending: Final = create(
+            model="openai/gpt-4o-mini",
+            max_tokens=80,
+            messages=[{"role": "user", "content": "hi"}],
+            stop_sequences=["STOPPROBE", "ENDPROBE"],
+            stream=stream,
+            api_key="test-api-key",
+        )
+        result: Final = await pending if is_async else pending
+        if stream:
+            chunks: Final = [chunk async for chunk in result] if is_async else list(result)
+            assert "message_stop" in "".join(chunk.decode() if isinstance(chunk, bytes) else chunk for chunk in chunks)
+        else:
+            assert result["content"] == [{"type": "text", "text": ""}]
+        body: Final = json.loads(chat.calls.last.request.content)
+        assert body["stop"] == ["STOPPROBE", "ENDPROBE"]
+        assert "stop_sequences" not in body
+
+
+@pytest.mark.parametrize("stop_sequences", [None, []], ids=["no-stops", "empty-stops"])
+def test_anthropic_experimental_pass_through_messages_handler(stop_sequences):
     """
     Test that api key is passed to litellm.responses for OpenAI models.
     OpenAI and Azure models are routed directly to the Responses API.
@@ -48,6 +99,7 @@ def test_anthropic_experimental_pass_through_messages_handler():
                 messages=[{"role": "user", "content": "Hello, how are you?"}],
                 model="openai/claude-3-5-sonnet-20240620",
                 api_key="test-api-key",
+                stop_sequences=stop_sequences,
             )
         except (ValueError, TypeError, AttributeError) as e:
             print(f"Error: {e}")