From aa33b052d2292c9fcdc3d34523a968f1b41d0bd4 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Mon, 26 Oct 2026 20:28:44 -0700 Subject: [PATCH] fix(messages): honor OpenAI stop_sequences (#37118) (#48534) --- .../pass_through/messages/handler.py | 5 +- ...erimental_pass_through_messages_handler.py | 54 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index a93e9106fb4..e1757fbc2d3 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -99,6 +99,7 @@ def _should_route_to_responses_api( requested_model: str | None = None, resolved_model: str | None = None, deployment_model_info: object = None, + stop_sequences: Sequence[str] | None = None, ) -> bool: """Return True when the request should use the Responses API path. @@ -107,6 +108,8 @@ def _should_route_to_responses_api( """ if litellm.use_chat_completions_url_for_anthropic_messages: return False + if stop_sequences: + return False if custom_llm_provider in _RESPONSES_API_PROVIDERS: return True if custom_llm_provider is None or requested_model is None or resolved_model is None: @@ -690,7 +693,7 @@ def anthropic_messages_handler( if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( - custom_llm_provider, original_model, model, kwargs.get("model_info") + custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences ): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index ff8b75af4d4..bb4388b9d0e 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -32,7 +32,58 @@ from litellm.utils import _invalidate_model_cost_lowercase_map import unittest.mock -def test_anthropic_experimental_pass_through_messages_handler(): +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"]) +async def test_openai_messages_forwards_stop_sequences_to_chat_completions( + monkeypatch: pytest.MonkeyPatch, is_async: bool, stream: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response: Final = ( + httpx.Response( + 200, + json={ + "id": "chatcmpl-stop-sequences", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1}, + }, + ) + if not stream + else httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text='data: {"id":"chatcmpl-stop-sequences","object":"chat.completion.chunk","created":1,' + '"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":""},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n", + ) + ) + with respx.mock as upstream: + chat: Final = upstream.post("https://api.openai.com/v1/chat/completions").mock(return_value=response) + create: Final = litellm.anthropic.messages.acreate if is_async else litellm.anthropic.messages.create + pending: Final = create( + model="openai/gpt-4o-mini", + max_tokens=80, + messages=[{"role": "user", "content": "hi"}], + stop_sequences=["STOPPROBE", "ENDPROBE"], + stream=stream, + api_key="test-api-key", + ) + result: Final = await pending if is_async else pending + if stream: + chunks: Final = [chunk async for chunk in result] if is_async else list(result) + assert "message_stop" in "".join(chunk.decode() if isinstance(chunk, bytes) else chunk for chunk in chunks) + else: + assert result["content"] == [{"type": "text", "text": ""}] + body: Final = json.loads(chat.calls.last.request.content) + assert body["stop"] == ["STOPPROBE", "ENDPROBE"] + assert "stop_sequences" not in body + + +@pytest.mark.parametrize("stop_sequences", [None, []], ids=["no-stops", "empty-stops"]) +def test_anthropic_experimental_pass_through_messages_handler(stop_sequences): """ Test that api key is passed to litellm.responses for OpenAI models. OpenAI and Azure models are routed directly to the Responses API. @@ -48,6 +99,7 @@ def test_anthropic_experimental_pass_through_messages_handler(): messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/claude-3-5-sonnet-20240620", api_key="test-api-key", + stop_sequences=stop_sequences, ) except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}")