fix(anthropic): include null stop_sequence in native responses (#47987)

* fix(anthropic): preserve nullable stop_sequence in native responses

* refactor(anthropic): clarify stop sequence defaults and test streaming

* refactor(anthropic): extract stop sequence normalization helper

* refactor(anthropic): default missing stop sequence in place
This commit is contained in:
Ishaan 2026-10-23 11:01:06 -07:00 • committed by GitHub
parent 08f04fbc97
commit e5cedbb861
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 108 additions and 4 deletions

View file

@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
_messages_carry_output_config = messages_carry_output_config
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
if "stop_sequence" not in response:
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
return response
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> AnthropicMessagesResponse:
"""
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
"""
try:
raw_response_json: Final = raw_response.json()
except Exception:
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
return AnthropicMessagesResponse(**raw_response_json)
response: Final = AnthropicMessagesResponse(**raw_response_json)
return _ensure_stop_sequence(response)
def get_async_streaming_response_iterator(
self,

View file

@ -2033,6 +2033,7 @@ async def test_anthropic_messages_with_extra_headers():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@ -2096,6 +2097,7 @@ async def test_anthropic_messages_with_thinking():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@ -2210,6 +2212,7 @@ async def test_anthropic_messages_bedrock_dynamic_region():
"content": [{"type": "text", "text": "This is a mock response"}],
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}

View file

@ -1,17 +1,23 @@
from collections.abc import Mapping
from datetime import datetime, timezone
from pathlib import Path
from typing import Final
from unittest.mock import patch
import httpx
import pytest
from pydantic import JsonValue
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.wif import get_anthropic_wif_token
from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig
from litellm.types.llms.anthropic import AnthropicFinishReason
from litellm.types.router import GenericLiteLLMParams
from tests.unit.llms.anthropic.test_anthropic_wif import (
ScriptedPoster,
@ -27,6 +33,52 @@ _WIF_PARAMS: Final[dict] = {
}
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
@pytest.mark.parametrize(
("stop_reason", "stop_fields", "expected_stop_sequence"),
(
pytest.param("end_turn", {}, None, id="end_turn_missing"),
pytest.param("max_tokens", {}, None, id="max_tokens_missing"),
pytest.param("tool_use", {}, None, id="tool_use_missing"),
pytest.param("end_turn", {"stop_sequence": None}, None, id="explicit_null"),
pytest.param("stop_sequence", {"stop_sequence": ", 7"}, ", 7", id="matched_stop"),
),
)
def test_native_response_defaults_stop_sequence_without_changing_other_fields(
config: AnthropicMessagesConfig,
stop_reason: AnthropicFinishReason,
stop_fields: Mapping[str, str | None],
expected_stop_sequence: str | None,
) -> None:
upstream: Final[dict[str, JsonValue]] = {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "test-model",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": stop_reason,
"usage": {"input_tokens": 19, "output_tokens": 42},
**stop_fields,
}
logging_obj: Final = Logging(
model="test-model",
messages=[],
stream=False,
call_type="anthropic_messages",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="test-call",
function_id="test-function",
)
response: Final = config.transform_anthropic_messages_response(
model="test-model", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj
)
assert response == {
**upstream,
"stop_sequence": expected_stop_sequence,
}
def test_workload_identity_allowed_for_anthropic() -> None:
assert AnthropicMessagesConfig()._allows_workload_identity is True

View file

@ -26,7 +26,9 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
is_provider_error_chunk,
parse_anthropic_error_event,
)
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.types.utils import Choices, Message, ModelResponse
@ -1733,6 +1735,49 @@ async def _drained(stream: AsyncIterator[bytes]) -> list[bytes]:
return [chunk async for chunk in stream]
@pytest.mark.asyncio
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
@pytest.mark.parametrize("stop_sequence", (None, ", 7"), ids=("null", "matched_stop"))
async def test_native_stream_preserves_nullable_stop_fields_across_chunk_boundaries(
config: AnthropicMessagesConfig, stop_sequence: str | None
) -> None:
logging_obj: Final = _make_logging_obj("native_stream_stop_fields")
events: Final = (
{
"type": "message_start",
"message": {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": logging_obj.model,
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 19, "output_tokens": 0},
},
},
{
"type": "message_delta",
"delta": {
"stop_reason": "end_turn" if stop_sequence is None else "stop_sequence",
"stop_sequence": stop_sequence,
},
"usage": {"output_tokens": 0},
},
{"type": "message_stop"},
)
upstream: Final = b"".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events)
response: Final = httpx.Response(200, content=_chunked(upstream, 7))
stream: Final = config.get_async_streaming_response_iterator(
model=logging_obj.model,
httpx_response=response,
request_body={"model": logging_obj.model},
litellm_logging_obj=logging_obj,
)
assert b"".join(await _drained(stream)) == upstream
def _data_events(frames: Sequence[bytes]) -> list[Mapping[str, object]]:
return [
json.loads(line[len(b"data:") :]) for frame in frames for line in frame.splitlines() if line.startswith(b"data:")