diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 6f6822a975b..75b0f93fd77 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -55,17 +55,14 @@ on: permissions: contents: read +env: + UV_PYTHON: "3.12" + jobs: run: - name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }} + name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.job-timeout-minutes }} - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - env: - UV_PYTHON: ${{ matrix.python-version }} permissions: contents: read pull-requests: read @@ -88,7 +85,7 @@ jobs: timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.UV_PYTHON }} - name: Set up uv if: steps.changes.outputs.decision != 'skip' @@ -103,9 +100,9 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.UV_CACHE_DIR }} - key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} + key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}- + ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}- - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' @@ -139,7 +136,7 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} DIST: ${{ inputs.dist }} - COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }} + COVERAGE_CORE: sysmon run: | if [ "${WORKERS}" = "0" ]; then uv run --no-sync pytest ${TEST_PATH:?} \ @@ -166,7 +163,7 @@ jobs: fi - name: Save coverage report - if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip' + if: always() && steps.changes.outputs.decision != 'skip' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 32b8cbe6343..9158ff4569f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -11,6 +11,7 @@ from typing import ( Final, Literal, Protocol, + cast, # noqa: TID251 # rebuilt message_delta dict spans the ContentBlockDelta/MessageBlockDelta union get_args, ) @@ -100,6 +101,10 @@ class _CombinedChunkSplitter: @staticmethod def _is_combined(chunk: "ModelResponseStream") -> bool: """True if ``chunk`` carries response content AND a finish_reason.""" + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + choices: Final = _optional_attr_sequence(chunk, "choices") if not choices: return False @@ -114,6 +119,7 @@ class _CombinedChunkSplitter: or _optional_attr(delta, "tool_calls") or _optional_attr(delta, "reasoning_content") or _optional_attr(delta, "thinking_blocks") + or openai_chat_refusal_text(delta) ) _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = ( @@ -305,6 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage + self._refusal_text: str = "" self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -572,6 +579,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): current_content_block_index=self.current_content_block_index, applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None), ) + processed_chunk = self._with_refusal_stop_details(processed_chunk) # Check if this is a usage chunk and we have a held stop_reason chunk if will_merge_into_held: @@ -806,6 +814,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): current_content_block_index=self.current_content_block_index, applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None), ) + processed_chunk = self._with_refusal_stop_details(processed_chunk) # Check if this is a usage chunk and we have a held stop_reason chunk if will_merge_into_held: @@ -993,6 +1002,31 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): def _increment_content_block_index(self): self.current_content_block_index += 1 + def _with_refusal_stop_details( + self, + processed_chunk: ContentBlockDelta | MessageBlockDelta, + ) -> ContentBlockDelta | MessageBlockDelta: + if processed_chunk.get("type") != "message_delta" or not self._refusal_text: + return processed_chunk + delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use + if delta.get("stop_reason") == "max_tokens": + return processed_chunk + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + ) + + return cast( # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch + ContentBlockDelta | MessageBlockDelta, + { # mutable-ok: fresh translation payload; never mutated after construction + **processed_chunk, + "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction + **delta, + "stop_reason": "refusal", + "stop_details": refusal_stop_details(self._refusal_text), + }, + }, + ) + @staticmethod def _delta_has_content(processed_chunk: Mapping[str, object]) -> bool: """Return True if a translated chunk carries a non-empty @@ -1035,6 +1069,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): @staticmethod def _is_blank_delta(chunk: "ModelResponseStream") -> bool: from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) choice: Final = chunk.choices[0] if choice.finish_reason is not None: @@ -1044,6 +1081,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return False if getattr(delta, "content", None): return False + if openai_chat_refusal_text(delta): + return False if getattr(delta, "reasoning_content", None): return False # thinking_blocks whose entries are all empty AND unsigned must not @@ -1067,13 +1106,19 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): - Different content types in the response - Specific markers in the content """ + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + from .transformation import LiteLLMAnthropicMessagesAdapter - # Example logic - customize based on your needs: - # If chunk indicates a tool call if chunk.choices[0].finish_reason is not None: return False + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) + if refusal_text is not None: + self._refusal_text = self._refusal_text + refusal_text + ( block_type, content_block_start, diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index f450f3899f4..db890662132 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -117,6 +117,10 @@ from litellm.llms.anthropic.common_utils import ( from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + refusal_stop_details, +) from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, AllAnthropicPassThroughMessageValues, @@ -1314,6 +1318,8 @@ class LiteLLMAnthropicMessagesAdapter: new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) + if (refusal_text := openai_chat_refusal_text(choice.message)) is not None: + new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) # Handle tool calls (in parallel to text content) if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: for tool_call in choice.message.tool_calls: @@ -1472,14 +1478,23 @@ class LiteLLMAnthropicMessagesAdapter: choices=response.choices, tool_name_mapping=tool_name_mapping, ) + refusal_text: Final = next( + (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None), + None, + ) if polyfill_result is not None and polyfill_result.compaction_block is not None: anthropic_content.insert(0, polyfill_result.compaction_block) ## extract finish reason - anthropic_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( + translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( openai_finish_reason=response.choices[0].finish_reason ) + anthropic_finish_reason: Final = ( + "refusal" + if refusal_text is not None and translated_finish_reason != "max_tokens" + else translated_finish_reason + ) # extract usage usage: Final[Usage] = getattr(response, "usage") anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) @@ -1501,6 +1516,7 @@ class LiteLLMAnthropicMessagesAdapter: usage=anthropic_usage, content=anthropic_content, stop_reason=anthropic_finish_reason, + stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None), ) applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None @@ -1541,7 +1557,9 @@ class LiteLLMAnthropicMessagesAdapter: "signature": thought_sig, } return "tool_use", cast("ContentBlockContentBlockDict", tool_block) - elif choice.delta.content is not None and len(choice.delta.content) > 0: + elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text( + choice.delta + ) is not None: return "text", TextBlock(type="text", text="") elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): thinking_blocks = choice.delta.thinking_blocks or [] @@ -1613,7 +1631,10 @@ class LiteLLMAnthropicMessagesAdapter: elif reasoning_content: return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content) else: - return "text_delta", ContentTextBlockDelta(type="text_delta", text=text) + refusal_text: Final = "".join( + refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None + ) + return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text) def translate_streaming_openai_response_to_anthropic( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index 242300c7b6d..7545dff1408 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -1,8 +1,11 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints -from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams +from litellm.types.llms.anthropic import ( + AnthropicMessagesRequestOptionalParams, + AnthropicStopDetails, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -25,6 +28,69 @@ def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] | return stop_details if isinstance(stop_details, dict) else None +def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails: + """The ``stop_details`` object accompanying a translated ``stop_reason: "refusal"``.""" + return AnthropicStopDetails(type="refusal", category=None, explanation=explanation) + + +def _mapping_field(container: object, key: str) -> object | None: + """One key of a raw provider payload, or None when the payload is not a mapping.""" + if not isinstance(container, Mapping): + return None + return cast(Mapping[str, object], container).get(key) # cast-ok: raw payload, callers re-check every value + + +def _mapping_str_field(container: object, key: str) -> str | None: + value: Final = _mapping_field(container, key) + return value if isinstance(value, str) and value else None + + +def openai_chat_refusal_text(message_or_delta: object) -> str | None: + """ + Refusal text carried by an OpenAI Chat Completions message or streaming delta, + read from ``refusal`` or from the ``provider_specific_fields`` LiteLLM parks it + in, or None when the turn is not a refusal. + """ + refusal: Final = getattr(message_or_delta, "refusal", None) + if isinstance(refusal, str) and refusal: + return refusal + return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal") + + +def _responses_message_refusal_text(item: object) -> str | None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + if isinstance(item, ResponseOutputMessage): + return next( + (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal) and part.refusal), + None, + ) + raw_parts: Final = _mapping_field(item, "content") + if _mapping_str_field(item, "type") != "message" or not isinstance(raw_parts, Sequence): + return None + return next( + ( + refusal + for part in cast(Sequence[object], raw_parts) # cast-ok: members re-validated below + if _mapping_str_field(part, "type") == "refusal" + and isinstance(refusal := _mapping_str_field(part, "refusal"), str) + ), + None, + ) + + +def responses_output_refusal_text(output: Iterable[object]) -> str | None: + """ + Refusal text carried by an OpenAI Responses ``output`` list, in typed + (``ResponseOutputRefusal``) or raw-dictionary shape, or None when none of the + output messages refused. + """ + return next( + (text for item in output if (text := _responses_message_refusal_text(item)) is not None), + None, + ) + + def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError": """The exception a safeguard-refused Anthropic response converts into so the content-policy fallback chain can re-dispatch it.""" diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index b0aba386753..2e0a6a9df8f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -1,13 +1,18 @@ # What is this? ## Translates OpenAI call to Anthropic `/v1/messages` format +import asyncio import json import traceback from collections import deque -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -49,6 +54,8 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque[dict[str, object]] = deque() + self._refusal_text: str = "" + self._sync_responses_iterator: Iterator[object] | None = None def _make_message_start(self) -> dict[str, object]: return { @@ -131,6 +138,24 @@ class AnthropicResponsesStreamWrapper: ) return + if event_type == "response.refusal.delta": + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + if not isinstance(delta, str) or not delta: + return + self._refusal_text = self._refusal_text + delta + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + block_idx = self._open_block(item_id, {"type": "text", "text": ""}) + self._chunk_queue.append( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + } + ) + return + # ---- text delta ---- if event_type == "response.output_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) @@ -215,34 +240,47 @@ class AnthropicResponsesStreamWrapper: response_obj: Final = getattr(event, "response", None) or ( event.get("response") if isinstance(event, dict) else None ) - stop_reason = "end_turn" - anthropic_usage: AnthropicUsage = AnthropicUsage(input_tokens=0, output_tokens=0) - - if response_obj is not None: - status: Final = getattr(response_obj, "status", None) - if status == "incomplete": - stop_reason = "max_tokens" - anthropic_usage = ( - LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( - getattr(response_obj, "usage", None) - ) + output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () + refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None) + status: Final = getattr(response_obj, "status", None) if response_obj is not None else None + has_tool_call: Final = any( + getattr(item, "type", None) == "function_call" + or (isinstance(item, dict) and item.get("type") == "function_call") + for item in output + ) + stop_reason: Final = ( + "max_tokens" + if status == "incomplete" + else "refusal" + if refusal_text is not None + else "tool_use" + if has_tool_call + else "end_turn" + ) + anthropic_usage: Final[AnthropicUsage] = ( + LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( + getattr(response_obj, "usage", None) ) + if response_obj is not None + else AnthropicUsage(input_tokens=0, output_tokens=0) + ) - # Check if tool_use was in the output to override stop_reason - if response_obj is not None: - output: Final = getattr(response_obj, "output", []) or [] - for out_item in output: - out_type = getattr(out_item, "type", None) or ( - out_item.get("type") if isinstance(out_item, dict) else None - ) - if out_type == "function_call": - stop_reason = "tool_use" - break + message_delta_payload: Final = { # mutable-ok: fresh message_delta payload built per chunk + "stop_reason": stop_reason, + "stop_sequence": None, + **( + { # mutable-ok: fresh message_delta stop_details entry built per chunk + "stop_details": refusal_stop_details(refusal_text) + } + if stop_reason == "refusal" + else {} # mutable-ok: empty spread placeholder for non-refusal stop + ), + } self._chunk_queue.append( { "type": "message_delta", - "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "delta": message_delta_payload, "usage": dict(anthropic_usage), } ) @@ -266,10 +304,20 @@ class AnthropicResponsesStreamWrapper: # Consume the upstream stream try: - async for event in self.responses_stream: - self._process_event(event) - if self._chunk_queue: - return self._chunk_queue.popleft() + if hasattr(self.responses_stream, "__aiter__"): + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + else: + if self._sync_responses_iterator is None: + self._sync_responses_iterator = iter(self.responses_stream) + sync_iterator: Final = self._sync_responses_iterator + missing: Final = object() + while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() except StopAsyncIteration: pass except Exception as e: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 62fc06a7c30..f1daf2be42a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -19,6 +19,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, prompt_cache_key_from_user_id, @@ -624,6 +628,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" + refusal_text: Final = responses_output_refusal_text( + cast(Iterable[object], response.output) # cast-ok: output items re-validated per item + ) for item in response.output: if isinstance(item, ResponseReasoningItem): @@ -631,10 +638,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(item, ResponseOutputMessage): for part in item.content: - if getattr(part, "type", None) == "output_text": + part_type = getattr(part, "type", None) + if part_type == "output_text": content.append( AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump() ) + elif part_type == "refusal": + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "refusal", "") or "" + ).model_dump() + ) elif isinstance(item, ResponseFunctionToolCall): try: @@ -654,11 +668,21 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(item, dict): item_type = item.get("type") if item_type == "message": - for part in item.get("content", []): - if isinstance(part, dict) and part.get("type") == "output_text": - content.append( - AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump() - ) + for part in item.get("content", ()): + if isinstance(part, dict): + part_type = part.get("type") + if part_type == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif part_type == "refusal": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("refusal", "") or "" + ).model_dump() + ) elif item_type == "reasoning": content.extend( self._thinking_blocks_from_reasoning_item( @@ -679,10 +703,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump(exclude_none=True) ) stop_reason = "tool_use" - - # status -> stop_reason override if response.status == "incomplete": stop_reason = "max_tokens" + elif refusal_text is not None: + stop_reason = "refusal" anthropic_usage: Final = self.translate_responses_api_usage_to_anthropic_usage(response.usage) @@ -695,4 +719,5 @@ class LiteLLMAnthropicToResponsesAPIAdapter: usage=anthropic_usage, content=content, stop_reason=stop_reason, + stop_details=(refusal_stop_details(refusal_text) if stop_reason == "refusal" else None), ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b3462203c4b..365d59a179b 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -520,8 +520,15 @@ ContentBlockContentBlockDict = ToolUseBlock | TextBlock | ChatCompletionThinking ContentBlockStart = ContentBlockStartToolUse | ContentBlockStartText +class AnthropicStopDetails(TypedDict, total=False): + type: ReadOnly[Literal["refusal"]] + category: ReadOnly[str | None] + explanation: ReadOnly[str | None] + + class MessageDelta(TypedDict, total=False): stop_reason: str | None + stop_details: ReadOnly[AnthropicStopDetails] class ServerToolUsage(TypedDict, total=False): @@ -658,7 +665,7 @@ class AnthropicOutputTokensDetails(BaseModel): thinking_tokens: int | None = None -AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] +AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] class AnthropicResponse(BaseModel): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 4fe1dafc73b..038a23a3ca2 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -5,6 +5,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, + AnthropicStopDetails, ContextManagementResponse, ServerToolUsage, ) @@ -78,16 +79,6 @@ class AnthropicUsage(TypedDict, total=False): server_tool_use: NotRequired[ReadOnly[ServerToolUsage]] -class AnthropicStopDetails(TypedDict, total=False): - """ - Safeguard verdict accompanying a `stop_reason: "refusal"` response: - https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback - """ - - category: ReadOnly[str | None] - explanation: ReadOnly[str | None] - - class AnthropicMessagesResponse(TypedDict, total=False): """ Anthropic Messages API Response: https://docs.anthropic.com/en/api/messages diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 0478a251057..30465ca25ba 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -40,6 +40,51 @@ from litellm.types.utils import ( ) +def test_translate_chat_refusal_to_anthropic_response(): + response = ModelResponse( + id="chatcmpl-refusal", + model="openai-model", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message(content=None, role="assistant", refusal="I cannot fulfill this request."), + ) + ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["content"] == [{"type": "text", "text": "I cannot fulfill this request."}] + assert result["stop_reason"] == "refusal" + assert result.get("stop_details") == { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + } + + +def test_translate_chat_length_takes_precedence_over_refusal(): + response = ModelResponse( + id="chatcmpl-partial-refusal", + model="openai-model", + choices=[ + Choices( + index=0, + finish_reason="length", + message=Message(content=None, role="assistant", refusal="Partial refusal"), + ) + ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["stop_reason"] == "max_tokens" + assert result.get("stop_details") is None + + def test_translate_streaming_openai_chunk_to_anthropic_content_block(): choices = [ StreamingChoices( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 17d42f55ae0..fdd08eaa182 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -108,6 +108,136 @@ def _text_deltas(events: List[dict]) -> List[str]: ] +def test_streaming_chat_refusal_emits_refusal_text_and_stop_details(): + chunks = [ + _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"] == { + "stop_reason": "refusal", + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + }, + } + + +@pytest.mark.asyncio +async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async(): + chunks = [ + _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted(): + """Providers that do not populate ``delta.refusal`` (Azure o-series among + them) hand LiteLLM the refusal as an unrecognized field, which lands in + ``provider_specific_fields``. That first delta still has to stream as text, + otherwise the client gets ``stop_reason: refusal`` over an empty content + array and shows the user nothing. + """ + chunks = [ + _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.asyncio +async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted_async(): + chunks = [ + _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): + """Fake-streamed responses arrive as one chunk carrying both the delta and the + finish_reason. The refusal has to be split off and streamed as text, or the + client gets ``stop_reason: refusal`` over an empty content array. + """ + chunks = [ + _make_chunk( + Delta(content=None, refusal="I cannot fulfill this request."), + finish_reason="stop", + ) + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.asyncio +async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async(): + chunks = [ + _make_chunk( + Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."}), + finish_reason="stop", + ) + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +async def test_streaming_chat_length_takes_precedence_over_refusal(async_mode: bool): + chunks = [ + _make_chunk(Delta(content=None, refusal="Partial refusal")), + _make_chunk(Delta(content=None), finish_reason="length"), + ] + stream = _AsyncStream(chunks) if async_mode else iter(chunks) + wrapper = AnthropicStreamWrapper(completion_stream=stream, model="openai-model") + + events = await _drain_async(wrapper) if async_mode else _drain_sync(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "max_tokens" + assert "stop_details" not in message_delta["delta"] + + def _input_json_deltas(events: List[dict]) -> List[str]: return [ e["delta"]["partial_json"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index aebbed88c70..d9df5df426a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -34,6 +34,14 @@ def _drain_async(events: list) -> list: return asyncio.run(_run()) +def _drain_sync_upstream(events: list) -> list: + async def _run() -> list: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter(events), model="m") + return [chunk async for chunk in wrapper] + + return asyncio.run(_run()) + + class TestMessageStartEmittedExactlyOnce: """The ``__anext__`` fallback emits ``message_start`` before consuming the stream, so ``_process_event`` must not emit a second one when @@ -55,6 +63,15 @@ class TestMessageStartEmittedExactlyOnce: chunks = _drain_async([{"type": "response.created"}]) assert chunks[0]["type"] == "message_start" + def test_sync_upstream_iterator_is_consumed(self): + chunks = _drain_sync_upstream( + [ + {"type": "response.created"}, + {"type": "response.output_text.delta", "item_id": "m1", "delta": "hi"}, + ] + ) + assert any(chunk.get("delta", {}).get("text") == "hi" for chunk in chunks) + class TestProcessEventResponseCreatedGuard: """``_process_event`` must emit ``message_start`` exactly once even if @@ -308,3 +325,60 @@ class TestResponseCompletedUsage: "cache_creation_input_tokens": 10, "cache_read_input_tokens": 4004, } + + +class TestRefusalStreamEvents: + def test_refusal_event_sequence_emits_refusal_text_and_stop_details(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}], + usage=None, + ) + chunks = _process_all( + [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.refusal.delta", "item_id": "msg_1", "delta": "I cannot fulfill this."}, + {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.completed", "response": response}, + ] + ) + assert [chunk["type"] for chunk in chunks] == [ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + assert chunks[2]["delta"] == {"type": "text_delta", "text": "I cannot fulfill this."} + assert chunks[4]["delta"] == { + "stop_reason": "refusal", + "stop_sequence": None, + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this.", + }, + } + + def test_response_completed_with_refusal_sets_stop_reason_refusal(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Policy violation"}]}], + usage=None, + ) + chunks = _process_all([{"type": "response.completed", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + + def test_incomplete_status_takes_precedence_over_refusal(self): + response = SimpleNamespace( + status="incomplete", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Partial refusal"}]}], + usage=None, + ) + chunks = _process_all([{"type": "response.incomplete", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "max_tokens" + assert "stop_details" not in message_delta["delta"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 0b2a9e655fd..9f8414afa38 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -147,9 +147,7 @@ class TestOutputConfigStructuredOutput: def test_output_config_format_explicit_strict_true_is_preserved(self): """Nested output_config.format with explicit strict=True is preserved.""" - req = _make_request( - output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}} - ) + req = _make_request(output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}}) kwargs = _ADAPTER.translate_request(req) assert kwargs["text"]["format"]["strict"] is True @@ -1207,6 +1205,18 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg +def _make_refusal_message(refusal_text: str): + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + return ResponseOutputMessage( + id="msg_refusal", + content=[ResponseOutputRefusal(type="refusal", refusal=refusal_text)], + role="assistant", + status="completed", + type="message", + ) + + def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: """Build a mock ResponseFunctionToolCall.""" from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] @@ -1280,6 +1290,41 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["stop_reason"] == "end_turn" + def test_refusal_part_becomes_text_block_and_sets_stop_reason_refusal(self): + response = _make_mock_response(output=[_make_refusal_message("I cannot fulfill this request.")]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "I cannot fulfill this request." + assert result["stop_reason"] == "refusal" + assert result.get("stop_details") == { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + } + + def test_dict_refusal_part_in_message_becomes_text_block(self): + output_item = { + "type": "message", + "content": [{"type": "refusal", "refusal": "Refused by policy"}], + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Refused by policy" + assert result["stop_reason"] == "refusal" + assert result.get("stop_details", {}).get("explanation") == "Refused by policy" + + def test_incomplete_status_takes_precedence_over_refusal(self): + response = _make_mock_response( + output=[_make_refusal_message("Partial refusal")], + status="incomplete", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "max_tokens" + assert result.get("stop_details") is None + def test_incomplete_status_sets_max_tokens(self): """status='incomplete' overrides stop_reason to 'max_tokens'.""" response = _make_mock_response( @@ -1338,9 +1383,7 @@ class TestTranslateResponse: ] ) result: Any = _ADAPTER.translate_response(response) - assert result["content"] == [ - {"type": "thinking", "thinking": "Weighing the options.", "signature": None} - ] + assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" @@ -1483,9 +1526,7 @@ class TestToolResultImages: }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}], }, ] @@ -1632,9 +1673,7 @@ class TestToolResultDocuments: }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}], }, ] @@ -1667,9 +1706,7 @@ class TestToolResultDocuments: def test_document_title_becomes_filename(self): output = self._tool_output(self._translate([self._base64_document(title="quarterly-report.pdf")])) - assert output == [ - {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI} - ] + assert output == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}] def test_url_document_becomes_file_url_part(self): output = self._tool_output( @@ -1778,9 +1815,7 @@ class TestUserContentDocuments: def test_document_title_becomes_filename(self): content = self._user_content(self._translate([self._base64_document(title="quarterly-report.pdf")])) - assert content == [ - {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI} - ] + assert content == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}] def test_url_document_becomes_file_url_part(self): content = self._user_content( @@ -1810,9 +1845,7 @@ class TestUserContentDocuments: assert content == [{"type": "input_text", "text": "still here"}] def test_document_breakpoint_rides_on_the_file_part(self): - content = self._user_content( - self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)]) - ) + content = self._user_content(self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)])) assert content == [ { "type": "input_file", @@ -1859,7 +1892,9 @@ class TestPromptCacheBreakpointToResponses: ] def test_system_without_breakpoint_still_becomes_instructions(self): - request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}]) + request = _make_request( + system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}] + ) kwargs = _ADAPTER.translate_request(request) assert kwargs["instructions"] == "Be concise.\nBe helpful." assert kwargs["input"] == [