diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index adf77dc3dd5..c6d744772a5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -1006,6 +1006,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": return processed_chunk + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + ) + return cast( # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch ContentBlockDelta | MessageBlockDelta, { # mutable-ok: fresh translation payload; never mutated after construction @@ -1013,11 +1017,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": { # mutable-ok: fresh stop_details payload; never mutated after construction - "type": "refusal", - "category": None, - "explanation": "".join(self._refusal_text_parts), - }, + "stop_details": refusal_stop_details("".join(self._refusal_text_parts)), }, }, ) @@ -1098,9 +1098,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): - Different content types in the response - Specific markers in the content """ + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + from .transformation import LiteLLMAnthropicMessagesAdapter - refusal_text: Final = LiteLLMAnthropicMessagesAdapter._refusal_text(chunk.choices[0].delta) + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) if refusal_text is not None: self._refusal_text_parts.append(refusal_text) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 215e3ded908..e3bad54d599 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -117,6 +117,10 @@ from litellm.llms.anthropic.common_utils import ( from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + refusal_stop_details, +) from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, AllAnthropicPassThroughMessageValues, @@ -308,17 +312,6 @@ class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass - @staticmethod - def _refusal_text(message_or_delta: object) -> str | None: - refusal: Final = getattr(message_or_delta, "refusal", None) - if isinstance(refusal, str): - return refusal - provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None) - if isinstance(provider_specific_fields, Mapping): - provider_refusal: Final = provider_specific_fields.get("refusal") - return provider_refusal if isinstance(provider_refusal, str) else None - return None - ### FOR [BETA] `/v1/messages` endpoint support def _extract_signature_from_tool_call(self, tool_call: object) -> str | None: @@ -1325,7 +1318,7 @@ class LiteLLMAnthropicMessagesAdapter: new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) - if (refusal_text := self._refusal_text(choice.message)) is not None: + if (refusal_text := openai_chat_refusal_text(choice.message)) is not None: new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) # Handle tool calls (in parallel to text content) if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: @@ -1486,7 +1479,7 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping=tool_name_mapping, ) refusal_text: Final = next( - (text for choice in response.choices if (text := self._refusal_text(choice.message)) is not None), + (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None), None, ) @@ -1523,15 +1516,7 @@ class LiteLLMAnthropicMessagesAdapter: usage=anthropic_usage, content=anthropic_content, stop_reason=anthropic_finish_reason, - stop_details=( - { # mutable-ok: fresh refusal stop_details payload built per response - "type": "refusal", - "category": None, - "explanation": refusal_text, - } - if anthropic_finish_reason == "refusal" - else None - ), + stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None), ) applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None @@ -1572,7 +1557,7 @@ class LiteLLMAnthropicMessagesAdapter: "signature": thought_sig, } return "tool_use", cast("ContentBlockContentBlockDict", tool_block) - elif (choice.delta.content is not None and len(choice.delta.content) > 0) or self._refusal_text( + elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text( choice.delta ) is not None: return "text", TextBlock(type="text", text="") @@ -1646,7 +1631,10 @@ class LiteLLMAnthropicMessagesAdapter: elif reasoning_content: return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content) else: - return "text_delta", ContentTextBlockDelta(type="text_delta", text=text) + refusal_text: Final = "".join( + refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None + ) + return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text) def translate_streaming_openai_response_to_anthropic( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index 242300c7b6d..7545dff1408 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -1,8 +1,11 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints -from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams +from litellm.types.llms.anthropic import ( + AnthropicMessagesRequestOptionalParams, + AnthropicStopDetails, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -25,6 +28,69 @@ def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] | return stop_details if isinstance(stop_details, dict) else None +def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails: + """The ``stop_details`` object accompanying a translated ``stop_reason: "refusal"``.""" + return AnthropicStopDetails(type="refusal", category=None, explanation=explanation) + + +def _mapping_field(container: object, key: str) -> object | None: + """One key of a raw provider payload, or None when the payload is not a mapping.""" + if not isinstance(container, Mapping): + return None + return cast(Mapping[str, object], container).get(key) # cast-ok: raw payload, callers re-check every value + + +def _mapping_str_field(container: object, key: str) -> str | None: + value: Final = _mapping_field(container, key) + return value if isinstance(value, str) and value else None + + +def openai_chat_refusal_text(message_or_delta: object) -> str | None: + """ + Refusal text carried by an OpenAI Chat Completions message or streaming delta, + read from ``refusal`` or from the ``provider_specific_fields`` LiteLLM parks it + in, or None when the turn is not a refusal. + """ + refusal: Final = getattr(message_or_delta, "refusal", None) + if isinstance(refusal, str) and refusal: + return refusal + return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal") + + +def _responses_message_refusal_text(item: object) -> str | None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + if isinstance(item, ResponseOutputMessage): + return next( + (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal) and part.refusal), + None, + ) + raw_parts: Final = _mapping_field(item, "content") + if _mapping_str_field(item, "type") != "message" or not isinstance(raw_parts, Sequence): + return None + return next( + ( + refusal + for part in cast(Sequence[object], raw_parts) # cast-ok: members re-validated below + if _mapping_str_field(part, "type") == "refusal" + and isinstance(refusal := _mapping_str_field(part, "refusal"), str) + ), + None, + ) + + +def responses_output_refusal_text(output: Iterable[object]) -> str | None: + """ + Refusal text carried by an OpenAI Responses ``output`` list, in typed + (``ResponseOutputRefusal``) or raw-dictionary shape, or None when none of the + output messages refused. + """ + return next( + (text for item in output if (text := _responses_message_refusal_text(item)) is not None), + None, + ) + + def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError": """The exception a safeguard-refused Anthropic response converts into so the content-policy fallback chain can re-dispatch it.""" diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 06ffcd26d40..5bd662e0f94 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,6 +9,10 @@ from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -136,8 +140,20 @@ class AnthropicResponsesStreamWrapper: if event_type == "response.refusal.delta": delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") - if isinstance(delta, str): - self._refusal_text_parts.append(delta) + if not isinstance(delta, str) or not delta: + return + self._refusal_text_parts.append(delta) + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + block_idx = self._open_block(item_id, {"type": "text", "text": ""}) + self._chunk_queue.append( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + } + ) return # ---- text delta ---- @@ -225,9 +241,7 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () - refusal_text: Final = LiteLLMAnthropicToResponsesAPIAdapter._refusal_text_from_output(output) or ( - "".join(self._refusal_text_parts) or None - ) + refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None) status: Final = getattr(response_obj, "status", None) if response_obj is not None else None has_tool_call: Final = any( getattr(item, "type", None) == "function_call" @@ -255,12 +269,8 @@ class AnthropicResponsesStreamWrapper: "stop_reason": stop_reason, "stop_sequence": None, **( - { # mutable-ok: fresh refusal stop_details payload built per chunk - "stop_details": { # mutable-ok: fresh refusal stop_details payload built per chunk - "type": "refusal", - "category": None, - "explanation": refusal_text, - } + { # mutable-ok: fresh message_delta stop_details entry built per chunk + "stop_details": refusal_stop_details(refusal_text) } if stop_reason == "refusal" else {} # mutable-ok: empty spread placeholder for non-refusal stop diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 318065148b8..9fb44127f5b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,7 +6,7 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping from itertools import groupby from typing import Any, Final, cast @@ -19,6 +19,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, prompt_cache_key_from_user_id, @@ -69,38 +73,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage) - @staticmethod - def _refusal_text_from_output(output: Iterable[object]) -> str | None: - from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal - - def refusal_text_from_item(item: object) -> str | None: - if isinstance(item, ResponseOutputMessage): - return next( - (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal)), - None, - ) - if not isinstance(item, Mapping): - return None - item_mapping: Final = cast(Mapping[str, object], item) # cast-ok: keys re-checked before use - raw_parts: Final = item_mapping.get("content") - if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): - return None - for part in cast(Sequence[object], raw_parts): # cast-ok: members re-validated below - if not isinstance(part, Mapping): - continue - part_mapping = cast(Mapping[str, object], part) # cast-ok: keys re-checked before use - if part_mapping.get("type") != "refusal": - continue - refusal = part_mapping.get("refusal") - if isinstance(refusal, str): - return refusal - return None - - return next( - (text for item in output if (text := refusal_text_from_item(item)) is not None), - None, - ) - # ------------------------------------------------------------------ # # Request translation: Anthropic -> Responses API # # ------------------------------------------------------------------ # @@ -656,7 +628,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" - refusal_text: Final = self._refusal_text_from_output( + refusal_text: Final = responses_output_refusal_text( cast(Iterable[object], response.output) # cast-ok: output items re-validated per item ) @@ -747,13 +719,5 @@ class LiteLLMAnthropicToResponsesAPIAdapter: usage=anthropic_usage, content=content, stop_reason=stop_reason, - stop_details=( - { # mutable-ok: fresh refusal stop_details payload built per response - "type": "refusal", - "category": None, - "explanation": refusal_text, - } - if stop_reason == "refusal" - else None - ), + stop_details=(refusal_stop_details(refusal_text) if stop_reason == "refusal" else None), ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index e7381986ec2..4359ace1870 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -108,7 +108,7 @@ def _text_deltas(events: List[dict]) -> List[str]: ] -def test_streaming_chat_refusal_emits_only_refusal_stop_details(): +def test_streaming_chat_refusal_emits_refusal_text_and_stop_details(): chunks = [ _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), _make_chunk(Delta(content=None), finish_reason="stop"), @@ -117,7 +117,7 @@ def test_streaming_chat_refusal_emits_only_refusal_stop_details(): events = _drain_sync(wrapper) - assert _text_deltas(events) == [] + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"] == { "stop_reason": "refusal", @@ -130,7 +130,7 @@ def test_streaming_chat_refusal_emits_only_refusal_stop_details(): @pytest.mark.asyncio -async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): +async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async(): chunks = [ _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), _make_chunk(Delta(content=None), finish_reason="stop"), @@ -139,7 +139,7 @@ async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): events = await _drain_async(wrapper) - assert _text_deltas(events) == [] + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index d8f16dde3e7..d9df5df426a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -328,7 +328,7 @@ class TestResponseCompletedUsage: class TestRefusalStreamEvents: - def test_refusal_event_sequence_emits_only_stop_details(self): + def test_refusal_event_sequence_emits_refusal_text_and_stop_details(self): response = SimpleNamespace( status="completed", output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}], @@ -346,11 +346,13 @@ class TestRefusalStreamEvents: assert [chunk["type"] for chunk in chunks] == [ "message_start", "content_block_start", + "content_block_delta", "content_block_stop", "message_delta", "message_stop", ] - assert chunks[3]["delta"] == { + assert chunks[2]["delta"] == {"type": "text_delta", "text": "I cannot fulfill this."} + assert chunks[4]["delta"] == { "stop_reason": "refusal", "stop_sequence": None, "stop_details": {