mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
fix(anthropic): include null stop_sequence in native responses (#47987)
* fix(anthropic): preserve nullable stop_sequence in native responses * refactor(anthropic): clarify stop sequence defaults and test streaming * refactor(anthropic): extract stop sequence normalization helper * refactor(anthropic): default missing stop sequence in place
This commit is contained in:
parent
08f04fbc97
commit
e5cedbb861
4 changed files with 108 additions and 4 deletions
|
|
@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
|
|||
_messages_carry_output_config = messages_carry_output_config
|
||||
|
||||
|
||||
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
|
||||
if "stop_sequence" not in response:
|
||||
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
|
||||
return response
|
||||
|
||||
|
||||
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
||||
_workload_identity_eligible: ClassVar[bool] = True
|
||||
|
||||
|
|
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> AnthropicMessagesResponse:
|
||||
"""
|
||||
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
|
||||
"""
|
||||
try:
|
||||
raw_response_json: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
return AnthropicMessagesResponse(**raw_response_json)
|
||||
response: Final = AnthropicMessagesResponse(**raw_response_json)
|
||||
return _ensure_stop_sequence(response)
|
||||
|
||||
def get_async_streaming_response_iterator(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2033,6 +2033,7 @@ async def test_anthropic_messages_with_extra_headers():
|
|||
],
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 20},
|
||||
}
|
||||
|
||||
|
|
@ -2096,6 +2097,7 @@ async def test_anthropic_messages_with_thinking():
|
|||
],
|
||||
"model": "claude-haiku-4-5-20251001",
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 20},
|
||||
}
|
||||
|
||||
|
|
@ -2210,6 +2212,7 @@ async def test_anthropic_messages_bedrock_dynamic_region():
|
|||
"content": [{"type": "text", "text": "This is a mock response"}],
|
||||
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 20},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,23 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.wif import get_anthropic_wif_token
|
||||
from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig
|
||||
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
|
||||
from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig
|
||||
from litellm.types.llms.anthropic import AnthropicFinishReason
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from tests.unit.llms.anthropic.test_anthropic_wif import (
|
||||
ScriptedPoster,
|
||||
|
|
@ -27,6 +33,52 @@ _WIF_PARAMS: Final[dict] = {
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
|
||||
@pytest.mark.parametrize(
|
||||
("stop_reason", "stop_fields", "expected_stop_sequence"),
|
||||
(
|
||||
pytest.param("end_turn", {}, None, id="end_turn_missing"),
|
||||
pytest.param("max_tokens", {}, None, id="max_tokens_missing"),
|
||||
pytest.param("tool_use", {}, None, id="tool_use_missing"),
|
||||
pytest.param("end_turn", {"stop_sequence": None}, None, id="explicit_null"),
|
||||
pytest.param("stop_sequence", {"stop_sequence": ", 7"}, ", 7", id="matched_stop"),
|
||||
),
|
||||
)
|
||||
def test_native_response_defaults_stop_sequence_without_changing_other_fields(
|
||||
config: AnthropicMessagesConfig,
|
||||
stop_reason: AnthropicFinishReason,
|
||||
stop_fields: Mapping[str, str | None],
|
||||
expected_stop_sequence: str | None,
|
||||
) -> None:
|
||||
upstream: Final[dict[str, JsonValue]] = {
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "test-model",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": stop_reason,
|
||||
"usage": {"input_tokens": 19, "output_tokens": 42},
|
||||
**stop_fields,
|
||||
}
|
||||
logging_obj: Final = Logging(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
litellm_call_id="test-call",
|
||||
function_id="test-function",
|
||||
)
|
||||
response: Final = config.transform_anthropic_messages_response(
|
||||
model="test-model", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj
|
||||
)
|
||||
|
||||
assert response == {
|
||||
**upstream,
|
||||
"stop_sequence": expected_stop_sequence,
|
||||
}
|
||||
|
||||
|
||||
def test_workload_identity_allowed_for_anthropic() -> None:
|
||||
assert AnthropicMessagesConfig()._allows_workload_identity is True
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,9 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
|
|||
is_provider_error_chunk,
|
||||
parse_anthropic_error_event,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
|
||||
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
|
||||
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
|
||||
|
|
@ -1733,6 +1735,49 @@ async def _drained(stream: AsyncIterator[bytes]) -> list[bytes]:
|
|||
return [chunk async for chunk in stream]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
|
||||
@pytest.mark.parametrize("stop_sequence", (None, ", 7"), ids=("null", "matched_stop"))
|
||||
async def test_native_stream_preserves_nullable_stop_fields_across_chunk_boundaries(
|
||||
config: AnthropicMessagesConfig, stop_sequence: str | None
|
||||
) -> None:
|
||||
logging_obj: Final = _make_logging_obj("native_stream_stop_fields")
|
||||
events: Final = (
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": logging_obj.model,
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 19, "output_tokens": 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {
|
||||
"stop_reason": "end_turn" if stop_sequence is None else "stop_sequence",
|
||||
"stop_sequence": stop_sequence,
|
||||
},
|
||||
"usage": {"output_tokens": 0},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
)
|
||||
upstream: Final = b"".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events)
|
||||
response: Final = httpx.Response(200, content=_chunked(upstream, 7))
|
||||
stream: Final = config.get_async_streaming_response_iterator(
|
||||
model=logging_obj.model,
|
||||
httpx_response=response,
|
||||
request_body={"model": logging_obj.model},
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
assert b"".join(await _drained(stream)) == upstream
|
||||
|
||||
|
||||
def _data_events(frames: Sequence[bytes]) -> list[Mapping[str, object]]:
|
||||
return [
|
||||
json.loads(line[len(b"data:") :]) for frame in frames for line in frame.splitlines() if line.startswith(b"data:")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue