From 200e2901d661d9bcea83d6e69c3839ea71b07e2c Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:33:10 +0530 Subject: [PATCH 1/8] fix(anthropic_responses): preserve Responses refusal blocks in Anthropic translation When OpenAI Responses returns a refusal content block, Anthropic /v1/messages erased the refusal text into an empty content array and emitted stop_reason 'end_turn'. Translate refusal blocks to Anthropic text blocks, map stop_reason to 'refusal', and add 'refusal' to AnthropicFinishReason. Fixes #39721 --- .../responses_adapters/streaming_iterator.py | 22 +++++- .../responses_adapters/transformation.py | 35 +++++++-- litellm/types/llms/anthropic.py | 2 +- ...t_responses_adapters_streaming_iterator.py | 34 +++++++++ .../test_responses_adapters_transformation.py | 76 +++++++++++++------ 5 files changed, 138 insertions(+), 31 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index a97ce18d179..5f6c5bad190 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -111,7 +111,7 @@ class AnthropicResponsesStreamWrapper: item_type: Final = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) - if item_type == "message": + if item_type in ("message", "refusal"): self._open_block(item_id, {"type": "text", "text": ""}) elif item_type == "function_call": call_id: Final = ( @@ -132,7 +132,7 @@ class AnthropicResponsesStreamWrapper: return # ---- text delta ---- - if event_type == "response.output_text.delta": + if event_type in ("response.output_text.delta", "response.refusal.delta"): item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index @@ -238,6 +238,24 @@ class AnthropicResponsesStreamWrapper: if out_type == "function_call": stop_reason = "tool_use" break + elif out_type == "refusal": + stop_reason = "refusal" + break + elif out_type == "message": + content_parts = getattr(out_item, "content", ()) or ( + out_item.get("content") or () if isinstance(out_item, dict) else () + ) + for part in content_parts: + part_type = getattr(part, "type", None) or ( + part.get("type") if isinstance(part, dict) else None + ) + if ( + part_type == "refusal" + or hasattr(part, "refusal") + or (isinstance(part, dict) and "refusal" in part) + ): + stop_reason = "refusal" + break self._chunk_queue.append( { diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 0eb0e38a46e..6cabd35117d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -631,10 +631,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(item, ResponseOutputMessage): for part in item.content: - if getattr(part, "type", None) == "output_text": + part_type = getattr(part, "type", None) + if part_type == "output_text": content.append( AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump() ) + elif part_type == "refusal" or hasattr(part, "refusal"): + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "refusal", "") or "" + ).model_dump() + ) + stop_reason = "refusal" elif isinstance(item, ResponseFunctionToolCall): try: @@ -654,11 +662,22 @@ class LiteLLMAnthropicToResponsesAPIAdapter: elif isinstance(item, dict): item_type = item.get("type") if item_type == "message": - for part in item.get("content", []): - if isinstance(part, dict) and part.get("type") == "output_text": - content.append( - AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump() - ) + for part in item.get("content", ()): + if isinstance(part, dict): + part_type = part.get("type") + if part_type == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif part_type == "refusal" or "refusal" in part: + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("refusal", "") or "" + ).model_dump() + ) + stop_reason = "refusal" elif item_type == "reasoning": content.extend( self._thinking_blocks_from_reasoning_item( @@ -679,6 +698,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) stop_reason = "tool_use" + elif item_type == "refusal": + refusal_text = item.get("refusal") or item.get("text", "") or "" + content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) + stop_reason = "refusal" # status -> stop_reason override if response.status == "incomplete": diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b3462203c4b..f1107c87c73 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -658,7 +658,7 @@ class AnthropicOutputTokensDetails(BaseModel): thinking_tokens: int | None = None -AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] +AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] class AnthropicResponse(BaseModel): diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index aebbed88c70..77ecfb73ff6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -308,3 +308,37 @@ class TestResponseCompletedUsage: "cache_creation_input_tokens": 10, "cache_read_input_tokens": 4004, } + + +class TestRefusalStreamEvents: + def test_refusal_delta_emits_text_delta(self): + chunks = _process_all( + [ + {"type": "response.created"}, + {"type": "response.refusal.delta", "item_id": "ref_1", "delta": "I cannot fulfill this."}, + ] + ) + assert any( + c.get("type") == "content_block_delta" and c.get("delta", {}).get("text") == "I cannot fulfill this." + for c in chunks + ) + + def test_response_completed_with_refusal_sets_stop_reason_refusal(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Policy violation"}]}], + usage=None, + ) + chunks = _process_all([{"type": "response.completed", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + + def test_response_completed_with_standalone_refusal_item_sets_stop_reason_refusal(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "refusal", "refusal": "Standalone refusal"}], + usage=None, + ) + chunks = _process_all([{"type": "response.completed", "response": response}]) + message_delta = next(c for c in chunks if c["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 5ecf604f096..f48c31ea5ed 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -147,9 +147,7 @@ class TestOutputConfigStructuredOutput: def test_output_config_format_explicit_strict_true_is_preserved(self): """Nested output_config.format with explicit strict=True is preserved.""" - req = _make_request( - output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}} - ) + req = _make_request(output_config={"format": {"type": "json_schema", "schema": self._SCHEMA, "strict": True}}) kwargs = _ADAPTER.translate_request(req) assert kwargs["text"]["format"]["strict"] is True @@ -1207,6 +1205,18 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg +def _make_refusal_message(refusal_text: str) -> MagicMock: + from openai.types.responses import ResponseOutputMessage + + part = MagicMock() + part.type = "refusal" + part.refusal = refusal_text + + msg = MagicMock(spec=ResponseOutputMessage) + msg.content = [part] + return msg + + def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: """Build a mock ResponseFunctionToolCall.""" from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] @@ -1279,6 +1289,38 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["stop_reason"] == "end_turn" + def test_refusal_part_becomes_text_block_and_sets_stop_reason_refusal(self): + response = _make_mock_response(output=[_make_refusal_message("I cannot fulfill this request.")]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "I cannot fulfill this request." + assert result["stop_reason"] == "refusal" + + def test_dict_refusal_part_in_message_becomes_text_block(self): + output_item = { + "type": "message", + "content": [{"type": "refusal", "refusal": "Refused by policy"}], + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Refused by policy" + assert result["stop_reason"] == "refusal" + + def test_dict_refusal_item_becomes_text_block(self): + output_item = { + "type": "refusal", + "refusal": "Standalone refusal", + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Standalone refusal" + assert result["stop_reason"] == "refusal" + def test_incomplete_status_sets_max_tokens(self): """status='incomplete' overrides stop_reason to 'max_tokens'.""" response = _make_mock_response( @@ -1337,9 +1379,7 @@ class TestTranslateResponse: ] ) result: Any = _ADAPTER.translate_response(response) - assert result["content"] == [ - {"type": "thinking", "thinking": "Weighing the options.", "signature": None} - ] + assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" @@ -1481,9 +1521,7 @@ class TestToolResultImages: }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}], }, ] @@ -1630,9 +1668,7 @@ class TestToolResultDocuments: }, { "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content} - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}], }, ] @@ -1665,9 +1701,7 @@ class TestToolResultDocuments: def test_document_title_becomes_filename(self): output = self._tool_output(self._translate([self._base64_document(title="quarterly-report.pdf")])) - assert output == [ - {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI} - ] + assert output == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}] def test_url_document_becomes_file_url_part(self): output = self._tool_output( @@ -1776,9 +1810,7 @@ class TestUserContentDocuments: def test_document_title_becomes_filename(self): content = self._user_content(self._translate([self._base64_document(title="quarterly-report.pdf")])) - assert content == [ - {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI} - ] + assert content == [{"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}] def test_url_document_becomes_file_url_part(self): content = self._user_content( @@ -1808,9 +1840,7 @@ class TestUserContentDocuments: assert content == [{"type": "input_text", "text": "still here"}] def test_document_breakpoint_rides_on_the_file_part(self): - content = self._user_content( - self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)]) - ) + content = self._user_content(self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)])) assert content == [ { "type": "input_file", @@ -1857,7 +1887,9 @@ class TestPromptCacheBreakpointToResponses: ] def test_system_without_breakpoint_still_becomes_instructions(self): - request = _make_request(system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}]) + request = _make_request( + system=[{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful."}] + ) kwargs = _ADAPTER.translate_request(request) assert kwargs["instructions"] == "Be concise.\nBe helpful." assert kwargs["input"] == [ From 0b34abe8fe3904453ff4796cfdb8b8981dfaf7e8 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:02:07 +0530 Subject: [PATCH 2/8] fix(anthropic): harden refusal translation --- .../adapters/streaming_iterator.py | 39 +++++- .../adapters/transformation.py | 37 +++++- .../responses_adapters/streaming_iterator.py | 114 +++++++++++------- .../responses_adapters/transformation.py | 58 +++++++-- litellm/types/llms/anthropic.py | 7 ++ .../anthropic_messages/anthropic_response.py | 11 +- ...al_pass_through_adapters_transformation.py | 45 +++++++ .../test_streaming_iterator_first_delta.py | 87 +++++++++++++ ...t_responses_adapters_streaming_iterator.py | 60 +++++++-- .../test_responses_adapters_transformation.py | 41 ++++--- 10 files changed, 397 insertions(+), 102 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 78ff83cafbf..41db4f12143 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -4,13 +4,14 @@ import copy import json import traceback from collections import deque -from collections.abc import AsyncIterator, Iterator, Sequence +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import ( TYPE_CHECKING, Any, Final, Literal, Protocol, + cast, get_args, ) @@ -305,6 +306,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage + self._refusal_text_parts: list[str] = [] self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -572,6 +574,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): current_content_block_index=self.current_content_block_index, applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None), ) + processed_chunk = self._with_refusal_stop_details(processed_chunk) # Check if this is a usage chunk and we have a held stop_reason chunk if will_merge_into_held: @@ -806,6 +809,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): current_content_block_index=self.current_content_block_index, applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None), ) + processed_chunk = self._with_refusal_stop_details(processed_chunk) # Check if this is a usage chunk and we have a held stop_reason chunk if will_merge_into_held: @@ -993,6 +997,31 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): def _increment_content_block_index(self): self.current_content_block_index += 1 + def _with_refusal_stop_details( + self, + processed_chunk: ContentBlockDelta | MessageBlockDelta, + ) -> ContentBlockDelta | MessageBlockDelta: + if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: + return processed_chunk + delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) + if delta.get("stop_reason") == "max_tokens": + return processed_chunk + return cast( + ContentBlockDelta | MessageBlockDelta, + { + **processed_chunk, + "delta": { + **delta, + "stop_reason": "refusal", + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "".join(self._refusal_text_parts), + }, + }, + }, + ) + @staticmethod def _delta_has_content(processed_chunk: dict[str, Any]) -> bool: """Return True if a translated chunk carries a non-empty @@ -1044,6 +1073,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return False if getattr(delta, "content", None): return False + if getattr(delta, "refusal", None): + return False if getattr(delta, "reasoning_content", None): return False # thinking_blocks whose entries are all empty AND unsigned must not @@ -1069,8 +1100,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): """ from .transformation import LiteLLMAnthropicMessagesAdapter - # Example logic - customize based on your needs: - # If chunk indicates a tool call + refusal_text: Final = LiteLLMAnthropicMessagesAdapter._refusal_text(chunk.choices[0].delta) + if refusal_text is not None: + self._refusal_text_parts.append(refusal_text) + if chunk.choices[0].finish_reason is not None: return False diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 573a461e89e..5655f59a4cc 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -307,6 +307,17 @@ class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass + @staticmethod + def _refusal_text(message_or_delta: object) -> str | None: + refusal: Final = getattr(message_or_delta, "refusal", None) + if isinstance(refusal, str): + return refusal + provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None) + if isinstance(provider_specific_fields, Mapping): + provider_refusal: Final = provider_specific_fields.get("refusal") + return provider_refusal if isinstance(provider_refusal, str) else None + return None + ### FOR [BETA] `/v1/messages` endpoint support def _extract_signature_from_tool_call(self, tool_call: object) -> str | None: @@ -1324,6 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter: new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) + if (refusal_text := self._refusal_text(choice.message)) is not None: + new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) # Handle tool calls (in parallel to text content) if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: for tool_call in choice.message.tool_calls: @@ -1482,14 +1495,23 @@ class LiteLLMAnthropicMessagesAdapter: choices=response.choices, tool_name_mapping=tool_name_mapping, ) + refusal_text: Final = next( + (text for choice in response.choices if (text := self._refusal_text(choice.message)) is not None), + None, + ) if polyfill_result is not None and polyfill_result.compaction_block is not None: anthropic_content.insert(0, polyfill_result.compaction_block) ## extract finish reason - anthropic_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( + translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic( openai_finish_reason=response.choices[0].finish_reason ) + anthropic_finish_reason: Final = ( + "refusal" + if refusal_text is not None and translated_finish_reason != "max_tokens" + else translated_finish_reason + ) # extract usage usage: Final[Usage] = getattr(response, "usage") anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) @@ -1511,6 +1533,15 @@ class LiteLLMAnthropicMessagesAdapter: usage=anthropic_usage, content=anthropic_content, stop_reason=anthropic_finish_reason, + stop_details=( + { + "type": "refusal", + "category": None, + "explanation": refusal_text, + } + if anthropic_finish_reason == "refusal" + else None + ), ) applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None @@ -1551,7 +1582,9 @@ class LiteLLMAnthropicMessagesAdapter: "signature": thought_sig, } return "tool_use", cast("ContentBlockContentBlockDict", tool_block) - elif choice.delta.content is not None and len(choice.delta.content) > 0: + elif (choice.delta.content is not None and len(choice.delta.content) > 0) or self._refusal_text( + choice.delta + ) is not None: return "text", TextBlock(type="text", text="") elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): thinking_blocks = choice.delta.thinking_blocks or [] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 5f6c5bad190..1b0ab5923d8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -1,5 +1,6 @@ # What is this? ## Translates OpenAI call to Anthropic `/v1/messages` format +import asyncio import json import traceback from collections import deque @@ -49,6 +50,8 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque = deque() + self._refusal_text_parts: list[str] = [] + self._sync_responses_iterator: Any = None def _make_message_start(self) -> dict[str, Any]: return { @@ -111,7 +114,7 @@ class AnthropicResponsesStreamWrapper: item_type: Final = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) - if item_type in ("message", "refusal"): + if item_type == "message": self._open_block(item_id, {"type": "text", "text": ""}) elif item_type == "function_call": call_id: Final = ( @@ -131,8 +134,14 @@ class AnthropicResponsesStreamWrapper: ) return + if event_type == "response.refusal.delta": + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + if isinstance(delta, str): + self._refusal_text_parts.append(delta) + return + # ---- text delta ---- - if event_type in ("response.output_text.delta", "response.refusal.delta"): + if event_type == "response.output_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index @@ -215,52 +224,53 @@ class AnthropicResponsesStreamWrapper: response_obj: Final = getattr(event, "response", None) or ( event.get("response") if isinstance(event, dict) else None ) - stop_reason = "end_turn" - anthropic_usage: AnthropicUsage = AnthropicUsage(input_tokens=0, output_tokens=0) - - if response_obj is not None: - status: Final = getattr(response_obj, "status", None) - if status == "incomplete": - stop_reason = "max_tokens" - anthropic_usage = ( - LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( - getattr(response_obj, "usage", None) - ) + output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () + refusal_text: Final = LiteLLMAnthropicToResponsesAPIAdapter._refusal_text_from_output(output) or ( + "".join(self._refusal_text_parts) or None + ) + status: Final = getattr(response_obj, "status", None) if response_obj is not None else None + has_tool_call: Final = any( + getattr(item, "type", None) == "function_call" + or (isinstance(item, dict) and item.get("type") == "function_call") + for item in output + ) + stop_reason: Final = ( + "max_tokens" + if status == "incomplete" + else "refusal" + if refusal_text is not None + else "tool_use" + if has_tool_call + else "end_turn" + ) + anthropic_usage: Final[AnthropicUsage] = ( + LiteLLMAnthropicToResponsesAPIAdapter.translate_responses_api_usage_to_anthropic_usage( + getattr(response_obj, "usage", None) ) + if response_obj is not None + else AnthropicUsage(input_tokens=0, output_tokens=0) + ) - # Check if tool_use was in the output to override stop_reason - if response_obj is not None: - output: Final = getattr(response_obj, "output", []) or [] - for out_item in output: - out_type = getattr(out_item, "type", None) or ( - out_item.get("type") if isinstance(out_item, dict) else None - ) - if out_type == "function_call": - stop_reason = "tool_use" - break - elif out_type == "refusal": - stop_reason = "refusal" - break - elif out_type == "message": - content_parts = getattr(out_item, "content", ()) or ( - out_item.get("content") or () if isinstance(out_item, dict) else () - ) - for part in content_parts: - part_type = getattr(part, "type", None) or ( - part.get("type") if isinstance(part, dict) else None - ) - if ( - part_type == "refusal" - or hasattr(part, "refusal") - or (isinstance(part, dict) and "refusal" in part) - ): - stop_reason = "refusal" - break + message_delta_payload: Final = { + "stop_reason": stop_reason, + "stop_sequence": None, + **( + { + "stop_details": { + "type": "refusal", + "category": None, + "explanation": refusal_text, + } + } + if stop_reason == "refusal" + else {} + ), + } self._chunk_queue.append( { "type": "message_delta", - "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "delta": message_delta_payload, "usage": dict(anthropic_usage), } ) @@ -284,10 +294,22 @@ class AnthropicResponsesStreamWrapper: # Consume the upstream stream try: - async for event in self.responses_stream: - self._process_event(event) - if self._chunk_queue: - return self._chunk_queue.popleft() + if hasattr(self.responses_stream, "__aiter__"): + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + else: + if self._sync_responses_iterator is None: + self._sync_responses_iterator = iter(self.responses_stream) + missing: Final = object() + while True: + event = await asyncio.to_thread(next, self._sync_responses_iterator, missing) + if event is missing: + break + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() except StopAsyncIteration: pass except Exception as e: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 6cabd35117d..92f3e08f24d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,7 +6,7 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from itertools import groupby from typing import Any, Final, cast @@ -69,6 +69,38 @@ class LiteLLMAnthropicToResponsesAPIAdapter: chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage) + @staticmethod + def _refusal_text_from_output(output: Iterable[object]) -> str | None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + def refusal_text_from_item(item: object) -> str | None: + if isinstance(item, ResponseOutputMessage): + return next( + (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal)), + None, + ) + if not isinstance(item, Mapping): + return None + item_mapping: Final = cast(Mapping[str, object], item) + raw_parts: Final = item_mapping.get("content") + if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): + return None + return next( + ( + refusal + for part in cast(Sequence[object], raw_parts) + if isinstance(part, Mapping) + and cast(Mapping[str, object], part).get("type") == "refusal" + and isinstance((refusal := cast(Mapping[str, object], part).get("refusal")), str) + ), + None, + ) + + return next( + (text for item in output if (text := refusal_text_from_item(item)) is not None), + None, + ) + # ------------------------------------------------------------------ # # Request translation: Anthropic -> Responses API # # ------------------------------------------------------------------ # @@ -624,6 +656,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" + refusal_text: Final = self._refusal_text_from_output(cast(Iterable[object], response.output)) for item in response.output: if isinstance(item, ResponseReasoningItem): @@ -636,13 +669,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content.append( AnthropicResponseContentBlockText(type="text", text=getattr(part, "text", "")).model_dump() ) - elif part_type == "refusal" or hasattr(part, "refusal"): + elif part_type == "refusal": content.append( AnthropicResponseContentBlockText( type="text", text=getattr(part, "refusal", "") or "" ).model_dump() ) - stop_reason = "refusal" elif isinstance(item, ResponseFunctionToolCall): try: @@ -671,13 +703,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: type="text", text=part.get("text", "") ).model_dump() ) - elif part_type == "refusal" or "refusal" in part: + elif part_type == "refusal": content.append( AnthropicResponseContentBlockText( type="text", text=part.get("refusal", "") or "" ).model_dump() ) - stop_reason = "refusal" elif item_type == "reasoning": content.extend( self._thinking_blocks_from_reasoning_item( @@ -698,14 +729,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) stop_reason = "tool_use" - elif item_type == "refusal": - refusal_text = item.get("refusal") or item.get("text", "") or "" - content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) - stop_reason = "refusal" - - # status -> stop_reason override if response.status == "incomplete": stop_reason = "max_tokens" + elif refusal_text is not None: + stop_reason = "refusal" anthropic_usage: Final = self.translate_responses_api_usage_to_anthropic_usage(response.usage) @@ -718,4 +745,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: usage=anthropic_usage, content=content, stop_reason=stop_reason, + stop_details=( + { + "type": "refusal", + "category": None, + "explanation": refusal_text, + } + if stop_reason == "refusal" + else None + ), ) diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index f1107c87c73..cdfc5227e06 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -520,8 +520,15 @@ ContentBlockContentBlockDict = ToolUseBlock | TextBlock | ChatCompletionThinking ContentBlockStart = ContentBlockStartToolUse | ContentBlockStartText +class AnthropicStopDetails(TypedDict, total=False): + type: ReadOnly[Literal["refusal"]] + category: ReadOnly[str | None] + explanation: ReadOnly[str | None] + + class MessageDelta(TypedDict, total=False): stop_reason: str | None + stop_details: AnthropicStopDetails class ServerToolUsage(TypedDict, total=False): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 4fe1dafc73b..038a23a3ca2 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -5,6 +5,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, + AnthropicStopDetails, ContextManagementResponse, ServerToolUsage, ) @@ -78,16 +79,6 @@ class AnthropicUsage(TypedDict, total=False): server_tool_use: NotRequired[ReadOnly[ServerToolUsage]] -class AnthropicStopDetails(TypedDict, total=False): - """ - Safeguard verdict accompanying a `stop_reason: "refusal"` response: - https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback - """ - - category: ReadOnly[str | None] - explanation: ReadOnly[str | None] - - class AnthropicMessagesResponse(TypedDict, total=False): """ Anthropic Messages API Response: https://docs.anthropic.com/en/api/messages diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2d74c00071b..fba5532d1e5 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -40,6 +40,51 @@ from litellm.types.utils import ( ) +def test_translate_chat_refusal_to_anthropic_response(): + response = ModelResponse( + id="chatcmpl-refusal", + model="openai-model", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=Message(content=None, role="assistant", refusal="I cannot fulfill this request."), + ) + ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["content"] == [{"type": "text", "text": "I cannot fulfill this request."}] + assert result["stop_reason"] == "refusal" + assert result.get("stop_details") == { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + } + + +def test_translate_chat_length_takes_precedence_over_refusal(): + response = ModelResponse( + id="chatcmpl-partial-refusal", + model="openai-model", + choices=[ + Choices( + index=0, + finish_reason="length", + message=Message(content=None, role="assistant", refusal="Partial refusal"), + ) + ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + ) + + result = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + + assert result["stop_reason"] == "max_tokens" + assert result.get("stop_details") is None + + def test_translate_streaming_openai_chunk_to_anthropic_content_block(): choices = [ StreamingChoices( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 17d42f55ae0..e7381986ec2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -108,6 +108,93 @@ def _text_deltas(events: List[dict]) -> List[str]: ] +def test_streaming_chat_refusal_emits_only_refusal_stop_details(): + chunks = [ + _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == [] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"] == { + "stop_reason": "refusal", + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + }, + } + + +@pytest.mark.asyncio +async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): + chunks = [ + _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == [] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): + chunks = [ + _make_chunk( + Delta(content=None, refusal="I cannot fulfill this request."), + finish_reason="stop", + ) + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.asyncio +async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async(): + chunks = [ + _make_chunk( + Delta(content=None, refusal="I cannot fulfill this request."), + finish_reason="stop", + ) + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +async def test_streaming_chat_length_takes_precedence_over_refusal(async_mode: bool): + chunks = [ + _make_chunk(Delta(content=None, refusal="Partial refusal")), + _make_chunk(Delta(content=None), finish_reason="length"), + ] + stream = _AsyncStream(chunks) if async_mode else iter(chunks) + wrapper = AnthropicStreamWrapper(completion_stream=stream, model="openai-model") + + events = await _drain_async(wrapper) if async_mode else _drain_sync(wrapper) + + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "max_tokens" + assert "stop_details" not in message_delta["delta"] + + def _input_json_deltas(events: List[dict]) -> List[str]: return [ e["delta"]["partial_json"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 77ecfb73ff6..d8f16dde3e7 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -34,6 +34,14 @@ def _drain_async(events: list) -> list: return asyncio.run(_run()) +def _drain_sync_upstream(events: list) -> list: + async def _run() -> list: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter(events), model="m") + return [chunk async for chunk in wrapper] + + return asyncio.run(_run()) + + class TestMessageStartEmittedExactlyOnce: """The ``__anext__`` fallback emits ``message_start`` before consuming the stream, so ``_process_event`` must not emit a second one when @@ -55,6 +63,15 @@ class TestMessageStartEmittedExactlyOnce: chunks = _drain_async([{"type": "response.created"}]) assert chunks[0]["type"] == "message_start" + def test_sync_upstream_iterator_is_consumed(self): + chunks = _drain_sync_upstream( + [ + {"type": "response.created"}, + {"type": "response.output_text.delta", "item_id": "m1", "delta": "hi"}, + ] + ) + assert any(chunk.get("delta", {}).get("text") == "hi" for chunk in chunks) + class TestProcessEventResponseCreatedGuard: """``_process_event`` must emit ``message_start`` exactly once even if @@ -311,17 +328,37 @@ class TestResponseCompletedUsage: class TestRefusalStreamEvents: - def test_refusal_delta_emits_text_delta(self): + def test_refusal_event_sequence_emits_only_stop_details(self): + response = SimpleNamespace( + status="completed", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}], + usage=None, + ) chunks = _process_all( [ {"type": "response.created"}, - {"type": "response.refusal.delta", "item_id": "ref_1", "delta": "I cannot fulfill this."}, + {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.refusal.delta", "item_id": "msg_1", "delta": "I cannot fulfill this."}, + {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, + {"type": "response.completed", "response": response}, ] ) - assert any( - c.get("type") == "content_block_delta" and c.get("delta", {}).get("text") == "I cannot fulfill this." - for c in chunks - ) + assert [chunk["type"] for chunk in chunks] == [ + "message_start", + "content_block_start", + "content_block_stop", + "message_delta", + "message_stop", + ] + assert chunks[3]["delta"] == { + "stop_reason": "refusal", + "stop_sequence": None, + "stop_details": { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this.", + }, + } def test_response_completed_with_refusal_sets_stop_reason_refusal(self): response = SimpleNamespace( @@ -333,12 +370,13 @@ class TestRefusalStreamEvents: message_delta = next(c for c in chunks if c["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" - def test_response_completed_with_standalone_refusal_item_sets_stop_reason_refusal(self): + def test_incomplete_status_takes_precedence_over_refusal(self): response = SimpleNamespace( - status="completed", - output=[{"type": "refusal", "refusal": "Standalone refusal"}], + status="incomplete", + output=[{"type": "message", "content": [{"type": "refusal", "refusal": "Partial refusal"}]}], usage=None, ) - chunks = _process_all([{"type": "response.completed", "response": response}]) + chunks = _process_all([{"type": "response.incomplete", "response": response}]) message_delta = next(c for c in chunks if c["type"] == "message_delta") - assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_reason"] == "max_tokens" + assert "stop_details" not in message_delta["delta"] diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index f48c31ea5ed..8205e993d26 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1205,16 +1205,16 @@ def _make_output_message(texts: List[str]) -> MagicMock: return msg -def _make_refusal_message(refusal_text: str) -> MagicMock: - from openai.types.responses import ResponseOutputMessage +def _make_refusal_message(refusal_text: str): + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal - part = MagicMock() - part.type = "refusal" - part.refusal = refusal_text - - msg = MagicMock(spec=ResponseOutputMessage) - msg.content = [part] - return msg + return ResponseOutputMessage( + id="msg_refusal", + content=[ResponseOutputRefusal(type="refusal", refusal=refusal_text)], + role="assistant", + status="completed", + type="message", + ) def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMock: @@ -1296,6 +1296,11 @@ class TestTranslateResponse: assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "I cannot fulfill this request." assert result["stop_reason"] == "refusal" + assert result.get("stop_details") == { + "type": "refusal", + "category": None, + "explanation": "I cannot fulfill this request.", + } def test_dict_refusal_part_in_message_becomes_text_block(self): output_item = { @@ -1308,18 +1313,16 @@ class TestTranslateResponse: assert result["content"][0]["type"] == "text" assert result["content"][0]["text"] == "Refused by policy" assert result["stop_reason"] == "refusal" + assert result.get("stop_details", {}).get("explanation") == "Refused by policy" - def test_dict_refusal_item_becomes_text_block(self): - output_item = { - "type": "refusal", - "refusal": "Standalone refusal", - } - response = _make_mock_response(output=[output_item]) + def test_incomplete_status_takes_precedence_over_refusal(self): + response = _make_mock_response( + output=[_make_refusal_message("Partial refusal")], + status="incomplete", + ) result: Any = _ADAPTER.translate_response(response) - assert len(result["content"]) == 1 - assert result["content"][0]["type"] == "text" - assert result["content"][0]["text"] == "Standalone refusal" - assert result["stop_reason"] == "refusal" + assert result["stop_reason"] == "max_tokens" + assert result.get("stop_details") is None def test_incomplete_status_sets_max_tokens(self): """status='incomplete' overrides stop_reason to 'max_tokens'.""" From 7399b3844dfc45ab0c9ddbb116b26235ce4f7df0 Mon Sep 17 00:00:00 2001 From: Atharva-Kanherkar <142440039+Atharva-Kanherkar@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:33:06 +0530 Subject: [PATCH 3/8] fix(anthropic): satisfy lint budget gates for refusal translation --- .../adapters/streaming_iterator.py | 14 +++++----- .../adapters/transformation.py | 2 +- .../responses_adapters/streaming_iterator.py | 10 +++---- .../responses_adapters/transformation.py | 28 ++++++++++--------- litellm/types/llms/anthropic.py | 2 +- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 41db4f12143..66fcad474d5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -11,7 +11,7 @@ from typing import ( Final, Literal, Protocol, - cast, + cast, # noqa: TID251 # rebuilt message_delta dict spans the ContentBlockDelta/MessageBlockDelta union get_args, ) @@ -306,7 +306,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage - self._refusal_text_parts: list[str] = [] + self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -1003,17 +1003,17 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): ) -> ContentBlockDelta | MessageBlockDelta: if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: return processed_chunk - delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) + delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": return processed_chunk - return cast( + return cast( # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch ContentBlockDelta | MessageBlockDelta, - { + { # mutable-ok: fresh translation payload; never mutated after construction **processed_chunk, - "delta": { + "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": { + "stop_details": { # mutable-ok: fresh stop_details payload; never mutated after construction "type": "refusal", "category": None, "explanation": "".join(self._refusal_text_parts), diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 5655f59a4cc..2d950f007a5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1534,7 +1534,7 @@ class LiteLLMAnthropicMessagesAdapter: content=anthropic_content, stop_reason=anthropic_finish_reason, stop_details=( - { + { # mutable-ok: fresh refusal stop_details payload built per response "type": "refusal", "category": None, "explanation": refusal_text, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 1b0ab5923d8..8e5ce77e48f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -50,7 +50,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque = deque() - self._refusal_text_parts: list[str] = [] + self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks self._sync_responses_iterator: Any = None def _make_message_start(self) -> dict[str, Any]: @@ -251,19 +251,19 @@ class AnthropicResponsesStreamWrapper: else AnthropicUsage(input_tokens=0, output_tokens=0) ) - message_delta_payload: Final = { + message_delta_payload: Final = { # mutable-ok: fresh message_delta payload built per chunk "stop_reason": stop_reason, "stop_sequence": None, **( - { - "stop_details": { + { # mutable-ok: fresh refusal stop_details payload built per chunk + "stop_details": { # mutable-ok: fresh refusal stop_details payload built per chunk "type": "refusal", "category": None, "explanation": refusal_text, } } if stop_reason == "refusal" - else {} + else {} # mutable-ok: empty spread placeholder for non-refusal stop ), } diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 92f3e08f24d..318065148b8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -81,20 +81,20 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) if not isinstance(item, Mapping): return None - item_mapping: Final = cast(Mapping[str, object], item) + item_mapping: Final = cast(Mapping[str, object], item) # cast-ok: keys re-checked before use raw_parts: Final = item_mapping.get("content") if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): return None - return next( - ( - refusal - for part in cast(Sequence[object], raw_parts) - if isinstance(part, Mapping) - and cast(Mapping[str, object], part).get("type") == "refusal" - and isinstance((refusal := cast(Mapping[str, object], part).get("refusal")), str) - ), - None, - ) + for part in cast(Sequence[object], raw_parts): # cast-ok: members re-validated below + if not isinstance(part, Mapping): + continue + part_mapping = cast(Mapping[str, object], part) # cast-ok: keys re-checked before use + if part_mapping.get("type") != "refusal": + continue + refusal = part_mapping.get("refusal") + if isinstance(refusal, str): + return refusal + return None return next( (text for item in output if (text := refusal_text_from_item(item)) is not None), @@ -656,7 +656,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" - refusal_text: Final = self._refusal_text_from_output(cast(Iterable[object], response.output)) + refusal_text: Final = self._refusal_text_from_output( + cast(Iterable[object], response.output) # cast-ok: output items re-validated per item + ) for item in response.output: if isinstance(item, ResponseReasoningItem): @@ -746,7 +748,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content=content, stop_reason=stop_reason, stop_details=( - { + { # mutable-ok: fresh refusal stop_details payload built per response "type": "refusal", "category": None, "explanation": refusal_text, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index cdfc5227e06..365d59a179b 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -528,7 +528,7 @@ class AnthropicStopDetails(TypedDict, total=False): class MessageDelta(TypedDict, total=False): stop_reason: str | None - stop_details: AnthropicStopDetails + stop_details: ReadOnly[AnthropicStopDetails] class ServerToolUsage(TypedDict, total=False): From 79d47788d9eea16c0038fc7eed10af37b5f8889b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:56:39 -0700 Subject: [PATCH 4/8] fix(anthropic): stream the refusal text on bridged /v1/messages calls Both bridges opened an empty text block on a refused streaming turn and closed it without a single delta, so a client replaying that assistant turn got HTTP 400 "text content blocks must be non-empty" from Anthropic. The safeguard-refusal fallback that motivated withholding the text only runs on the awaited non-streaming response, so nothing needed it withheld Move the refusal readers into the shared messages/utils helpers so the adapters stop reaching into each other's private statics, which is also what put reportPrivateUsage over its budget --- .../adapters/streaming_iterator.py | 16 +++-- .../adapters/transformation.py | 36 ++++------ .../messages/utils.py | 70 ++++++++++++++++++- .../responses_adapters/streaming_iterator.py | 32 ++++++--- .../responses_adapters/transformation.py | 50 ++----------- .../test_streaming_iterator_first_delta.py | 8 +-- ...t_responses_adapters_streaming_iterator.py | 6 +- 7 files changed, 126 insertions(+), 92 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index adf77dc3dd5..c6d744772a5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -1006,6 +1006,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": return processed_chunk + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + ) + return cast( # cast-ok: rebuilt dict matches the message_delta TypedDict shape for this branch ContentBlockDelta | MessageBlockDelta, { # mutable-ok: fresh translation payload; never mutated after construction @@ -1013,11 +1017,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": { # mutable-ok: fresh stop_details payload; never mutated after construction - "type": "refusal", - "category": None, - "explanation": "".join(self._refusal_text_parts), - }, + "stop_details": refusal_stop_details("".join(self._refusal_text_parts)), }, }, ) @@ -1098,9 +1098,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): - Different content types in the response - Specific markers in the content """ + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + from .transformation import LiteLLMAnthropicMessagesAdapter - refusal_text: Final = LiteLLMAnthropicMessagesAdapter._refusal_text(chunk.choices[0].delta) + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) if refusal_text is not None: self._refusal_text_parts.append(refusal_text) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 215e3ded908..e3bad54d599 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -117,6 +117,10 @@ from litellm.llms.anthropic.common_utils import ( from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + refusal_stop_details, +) from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, AllAnthropicPassThroughMessageValues, @@ -308,17 +312,6 @@ class LiteLLMAnthropicMessagesAdapter: def __init__(self): pass - @staticmethod - def _refusal_text(message_or_delta: object) -> str | None: - refusal: Final = getattr(message_or_delta, "refusal", None) - if isinstance(refusal, str): - return refusal - provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None) - if isinstance(provider_specific_fields, Mapping): - provider_refusal: Final = provider_specific_fields.get("refusal") - return provider_refusal if isinstance(provider_refusal, str) else None - return None - ### FOR [BETA] `/v1/messages` endpoint support def _extract_signature_from_tool_call(self, tool_call: object) -> str | None: @@ -1325,7 +1318,7 @@ class LiteLLMAnthropicMessagesAdapter: new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) - if (refusal_text := self._refusal_text(choice.message)) is not None: + if (refusal_text := openai_chat_refusal_text(choice.message)) is not None: new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump()) # Handle tool calls (in parallel to text content) if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0: @@ -1486,7 +1479,7 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping=tool_name_mapping, ) refusal_text: Final = next( - (text for choice in response.choices if (text := self._refusal_text(choice.message)) is not None), + (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None), None, ) @@ -1523,15 +1516,7 @@ class LiteLLMAnthropicMessagesAdapter: usage=anthropic_usage, content=anthropic_content, stop_reason=anthropic_finish_reason, - stop_details=( - { # mutable-ok: fresh refusal stop_details payload built per response - "type": "refusal", - "category": None, - "explanation": refusal_text, - } - if anthropic_finish_reason == "refusal" - else None - ), + stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None), ) applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None @@ -1572,7 +1557,7 @@ class LiteLLMAnthropicMessagesAdapter: "signature": thought_sig, } return "tool_use", cast("ContentBlockContentBlockDict", tool_block) - elif (choice.delta.content is not None and len(choice.delta.content) > 0) or self._refusal_text( + elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text( choice.delta ) is not None: return "text", TextBlock(type="text", text="") @@ -1646,7 +1631,10 @@ class LiteLLMAnthropicMessagesAdapter: elif reasoning_content: return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content) else: - return "text_delta", ContentTextBlockDelta(type="text_delta", text=text) + refusal_text: Final = "".join( + refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None + ) + return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text) def translate_streaming_openai_response_to_anthropic( self, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py index 242300c7b6d..7545dff1408 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/utils.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/utils.py @@ -1,8 +1,11 @@ -from collections.abc import Mapping +from collections.abc import Iterable, Mapping, Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints -from litellm.types.llms.anthropic import AnthropicMessagesRequestOptionalParams +from litellm.types.llms.anthropic import ( + AnthropicMessagesRequestOptionalParams, + AnthropicStopDetails, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -25,6 +28,69 @@ def get_safeguard_refusal_stop_details(response: object) -> Mapping[str, Any] | return stop_details if isinstance(stop_details, dict) else None +def refusal_stop_details(explanation: str | None) -> AnthropicStopDetails: + """The ``stop_details`` object accompanying a translated ``stop_reason: "refusal"``.""" + return AnthropicStopDetails(type="refusal", category=None, explanation=explanation) + + +def _mapping_field(container: object, key: str) -> object | None: + """One key of a raw provider payload, or None when the payload is not a mapping.""" + if not isinstance(container, Mapping): + return None + return cast(Mapping[str, object], container).get(key) # cast-ok: raw payload, callers re-check every value + + +def _mapping_str_field(container: object, key: str) -> str | None: + value: Final = _mapping_field(container, key) + return value if isinstance(value, str) and value else None + + +def openai_chat_refusal_text(message_or_delta: object) -> str | None: + """ + Refusal text carried by an OpenAI Chat Completions message or streaming delta, + read from ``refusal`` or from the ``provider_specific_fields`` LiteLLM parks it + in, or None when the turn is not a refusal. + """ + refusal: Final = getattr(message_or_delta, "refusal", None) + if isinstance(refusal, str) and refusal: + return refusal + return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal") + + +def _responses_message_refusal_text(item: object) -> str | None: + from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal + + if isinstance(item, ResponseOutputMessage): + return next( + (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal) and part.refusal), + None, + ) + raw_parts: Final = _mapping_field(item, "content") + if _mapping_str_field(item, "type") != "message" or not isinstance(raw_parts, Sequence): + return None + return next( + ( + refusal + for part in cast(Sequence[object], raw_parts) # cast-ok: members re-validated below + if _mapping_str_field(part, "type") == "refusal" + and isinstance(refusal := _mapping_str_field(part, "refusal"), str) + ), + None, + ) + + +def responses_output_refusal_text(output: Iterable[object]) -> str | None: + """ + Refusal text carried by an OpenAI Responses ``output`` list, in typed + (``ResponseOutputRefusal``) or raw-dictionary shape, or None when none of the + output messages refused. + """ + return next( + (text for item in output if (text := _responses_message_refusal_text(item)) is not None), + None, + ) + + def safeguard_refusal_error(model: str, stop_details: Mapping[str, object]) -> "ContentPolicyViolationError": """The exception a safeguard-refused Anthropic response converts into so the content-policy fallback chain can re-dispatch it.""" diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 06ffcd26d40..5bd662e0f94 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,6 +9,10 @@ from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -136,8 +140,20 @@ class AnthropicResponsesStreamWrapper: if event_type == "response.refusal.delta": delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") - if isinstance(delta, str): - self._refusal_text_parts.append(delta) + if not isinstance(delta, str) or not delta: + return + self._refusal_text_parts.append(delta) + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + if block_idx < 0: + block_idx = self._open_block(item_id, {"type": "text", "text": ""}) + self._chunk_queue.append( + { + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + } + ) return # ---- text delta ---- @@ -225,9 +241,7 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () - refusal_text: Final = LiteLLMAnthropicToResponsesAPIAdapter._refusal_text_from_output(output) or ( - "".join(self._refusal_text_parts) or None - ) + refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None) status: Final = getattr(response_obj, "status", None) if response_obj is not None else None has_tool_call: Final = any( getattr(item, "type", None) == "function_call" @@ -255,12 +269,8 @@ class AnthropicResponsesStreamWrapper: "stop_reason": stop_reason, "stop_sequence": None, **( - { # mutable-ok: fresh refusal stop_details payload built per chunk - "stop_details": { # mutable-ok: fresh refusal stop_details payload built per chunk - "type": "refusal", - "category": None, - "explanation": refusal_text, - } + { # mutable-ok: fresh message_delta stop_details entry built per chunk + "stop_details": refusal_stop_details(refusal_text) } if stop_reason == "refusal" else {} # mutable-ok: empty spread placeholder for non-refusal stop diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 318065148b8..9fb44127f5b 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -6,7 +6,7 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping from itertools import groupby from typing import Any, Final, cast @@ -19,6 +19,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + refusal_stop_details, + responses_output_refusal_text, +) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, prompt_cache_key_from_user_id, @@ -69,38 +73,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(raw_usage) return LiteLLMAnthropicMessagesAdapter._translate_openai_usage_to_anthropic_usage(chat_usage) - @staticmethod - def _refusal_text_from_output(output: Iterable[object]) -> str | None: - from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal - - def refusal_text_from_item(item: object) -> str | None: - if isinstance(item, ResponseOutputMessage): - return next( - (part.refusal for part in item.content if isinstance(part, ResponseOutputRefusal)), - None, - ) - if not isinstance(item, Mapping): - return None - item_mapping: Final = cast(Mapping[str, object], item) # cast-ok: keys re-checked before use - raw_parts: Final = item_mapping.get("content") - if item_mapping.get("type") != "message" or not isinstance(raw_parts, Sequence): - return None - for part in cast(Sequence[object], raw_parts): # cast-ok: members re-validated below - if not isinstance(part, Mapping): - continue - part_mapping = cast(Mapping[str, object], part) # cast-ok: keys re-checked before use - if part_mapping.get("type") != "refusal": - continue - refusal = part_mapping.get("refusal") - if isinstance(refusal, str): - return refusal - return None - - return next( - (text for item in output if (text := refusal_text_from_item(item)) is not None), - None, - ) - # ------------------------------------------------------------------ # # Request translation: Anthropic -> Responses API # # ------------------------------------------------------------------ # @@ -656,7 +628,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: content: Final[list[dict[str, object]]] = [] stop_reason: AnthropicFinishReason = "end_turn" - refusal_text: Final = self._refusal_text_from_output( + refusal_text: Final = responses_output_refusal_text( cast(Iterable[object], response.output) # cast-ok: output items re-validated per item ) @@ -747,13 +719,5 @@ class LiteLLMAnthropicToResponsesAPIAdapter: usage=anthropic_usage, content=content, stop_reason=stop_reason, - stop_details=( - { # mutable-ok: fresh refusal stop_details payload built per response - "type": "refusal", - "category": None, - "explanation": refusal_text, - } - if stop_reason == "refusal" - else None - ), + stop_details=(refusal_stop_details(refusal_text) if stop_reason == "refusal" else None), ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index e7381986ec2..4359ace1870 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -108,7 +108,7 @@ def _text_deltas(events: List[dict]) -> List[str]: ] -def test_streaming_chat_refusal_emits_only_refusal_stop_details(): +def test_streaming_chat_refusal_emits_refusal_text_and_stop_details(): chunks = [ _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), _make_chunk(Delta(content=None), finish_reason="stop"), @@ -117,7 +117,7 @@ def test_streaming_chat_refusal_emits_only_refusal_stop_details(): events = _drain_sync(wrapper) - assert _text_deltas(events) == [] + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"] == { "stop_reason": "refusal", @@ -130,7 +130,7 @@ def test_streaming_chat_refusal_emits_only_refusal_stop_details(): @pytest.mark.asyncio -async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): +async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async(): chunks = [ _make_chunk(Delta(content=None, refusal="I cannot fulfill this request.")), _make_chunk(Delta(content=None), finish_reason="stop"), @@ -139,7 +139,7 @@ async def test_streaming_chat_refusal_emits_only_refusal_stop_details_async(): events = await _drain_async(wrapper) - assert _text_deltas(events) == [] + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index d8f16dde3e7..d9df5df426a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -328,7 +328,7 @@ class TestResponseCompletedUsage: class TestRefusalStreamEvents: - def test_refusal_event_sequence_emits_only_stop_details(self): + def test_refusal_event_sequence_emits_refusal_text_and_stop_details(self): response = SimpleNamespace( status="completed", output=[{"type": "message", "content": [{"type": "refusal", "refusal": "I cannot fulfill this."}]}], @@ -346,11 +346,13 @@ class TestRefusalStreamEvents: assert [chunk["type"] for chunk in chunks] == [ "message_start", "content_block_start", + "content_block_delta", "content_block_stop", "message_delta", "message_stop", ] - assert chunks[3]["delta"] == { + assert chunks[2]["delta"] == {"type": "text_delta", "text": "I cannot fulfill this."} + assert chunks[4]["delta"] == { "stop_reason": "refusal", "stop_sequence": None, "stop_details": { From b21315fe754ddcb54b6e87759b621e5e17e515ef Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:36:43 -0700 Subject: [PATCH 5/8] refactor(anthropic): type the responses refusal stream iterator state Types the cached sync upstream iterator instead of holding it as Any, so the Responses to Anthropic streaming wrapper carries no untyped state. --- .../responses_adapters/streaming_iterator.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 5bd662e0f94..bcae732ad42 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -4,7 +4,7 @@ import asyncio import json import traceback from collections import deque -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger @@ -55,7 +55,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_stop = False self._chunk_queue: deque[dict[str, object]] = deque() self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks - self._sync_responses_iterator: Any = None + self._sync_responses_iterator: Iterator[object] | None = None def _make_message_start(self) -> dict[str, object]: return { @@ -312,11 +312,9 @@ class AnthropicResponsesStreamWrapper: else: if self._sync_responses_iterator is None: self._sync_responses_iterator = iter(self.responses_stream) + sync_iterator: Final = self._sync_responses_iterator missing: Final = object() - while True: - event = await asyncio.to_thread(next, self._sync_responses_iterator, missing) - if event is missing: - break + while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing: self._process_event(event) if self._chunk_queue: return self._chunk_queue.popleft() From c09d34fc4b84a619f10eb7e64fe44fea198a8dc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:20:19 -0700 Subject: [PATCH 6/8] fix(anthropic): stream refusals parked in provider_specific_fields The first-delta guard read `delta.refusal` directly, while the translation three lines later goes through `openai_chat_refusal_text`, which also reads the `provider_specific_fields` LiteLLM parks unrecognized fields in. A provider that sends the refusal that way had its only refusal delta skipped as blank, so the client got `stop_reason: refusal` over an empty content array, which is the symptom this PR set out to fix --- .../adapters/streaming_iterator.py | 5 ++- .../test_streaming_iterator_first_delta.py | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index c6d744772a5..b2513a5c046 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -1064,6 +1064,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): @staticmethod def _is_blank_delta(chunk: "ModelResponseStream") -> bool: from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) choice: Final = chunk.choices[0] if choice.finish_reason is not None: @@ -1073,7 +1076,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): return False if getattr(delta, "content", None): return False - if getattr(delta, "refusal", None): + if openai_chat_refusal_text(delta): return False if getattr(delta, "reasoning_content", None): return False diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 4359ace1870..4c5e2101fbf 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -145,6 +145,43 @@ async def test_streaming_chat_refusal_emits_refusal_text_and_stop_details_async( assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." +def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted(): + """Providers that do not populate ``delta.refusal`` (Azure o-series among + them) hand LiteLLM the refusal as an unrecognized field, which lands in + ``provider_specific_fields``. That first delta still has to stream as text, + otherwise the client gets ``stop_reason: refusal`` over an empty content + array and shows the user nothing. + """ + chunks = [ + _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=iter(chunks), model="openai-model") + + events = _drain_sync(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + +@pytest.mark.asyncio +async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emitted_async(): + chunks = [ + _make_chunk(Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."})), + _make_chunk(Delta(content=None), finish_reason="stop"), + ] + wrapper = AnthropicStreamWrapper(completion_stream=_AsyncStream(chunks), model="openai-model") + + events = await _drain_async(wrapper) + + assert _text_deltas(events) == ["I cannot fulfill this request."] + message_delta = next(event for event in events if event["type"] == "message_delta") + assert message_delta["delta"]["stop_reason"] == "refusal" + assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." + + def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): chunks = [ _make_chunk( From 05cba21763ee40477c2b4b13135f10763bb89957 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:40:20 -0700 Subject: [PATCH 7/8] fix(anthropic): split refusal off a combined finish_reason chunk A fake-streamed provider hands the adapter one chunk carrying both the delta payload and the finish_reason, which is exactly what the combined chunk splitter exists for, but its content check never listed the refusal. The translation short-circuits on finish_reason, so that refusal text was dropped and the client got `stop_reason: refusal` over an empty content array, the symptom this PR set out to fix. Both refusal accumulators also drop their `mutable-ok` lists for a plain string attribute --- .../adapters/streaming_iterator.py | 19 ++++++++++++------- .../responses_adapters/streaming_iterator.py | 6 +++--- .../test_streaming_iterator_first_delta.py | 8 +++++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index b2513a5c046..9158ff4569f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -101,6 +101,10 @@ class _CombinedChunkSplitter: @staticmethod def _is_combined(chunk: "ModelResponseStream") -> bool: """True if ``chunk`` carries response content AND a finish_reason.""" + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + choices: Final = _optional_attr_sequence(chunk, "choices") if not choices: return False @@ -115,6 +119,7 @@ class _CombinedChunkSplitter: or _optional_attr(delta, "tool_calls") or _optional_attr(delta, "reasoning_content") or _optional_attr(delta, "thinking_blocks") + or openai_chat_refusal_text(delta) ) _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = ( @@ -306,7 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage - self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks + self._refusal_text: str = "" self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -1001,7 +1006,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): self, processed_chunk: ContentBlockDelta | MessageBlockDelta, ) -> ContentBlockDelta | MessageBlockDelta: - if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: + if processed_chunk.get("type") != "message_delta" or not self._refusal_text: return processed_chunk delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": @@ -1017,7 +1022,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": refusal_stop_details("".join(self._refusal_text_parts)), + "stop_details": refusal_stop_details(self._refusal_text), }, }, ) @@ -1107,13 +1112,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): from .transformation import LiteLLMAnthropicMessagesAdapter - refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) - if refusal_text is not None: - self._refusal_text_parts.append(refusal_text) - if chunk.choices[0].finish_reason is not None: return False + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) + if refusal_text is not None: + self._refusal_text = self._refusal_text + refusal_text + ( block_type, content_block_start, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index bcae732ad42..2e0a6a9df8f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -54,7 +54,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque[dict[str, object]] = deque() - self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks + self._refusal_text: str = "" self._sync_responses_iterator: Iterator[object] | None = None def _make_message_start(self) -> dict[str, object]: @@ -142,7 +142,7 @@ class AnthropicResponsesStreamWrapper: delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") if not isinstance(delta, str) or not delta: return - self._refusal_text_parts.append(delta) + self._refusal_text = self._refusal_text + delta item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: @@ -241,7 +241,7 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () - refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None) + refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None) status: Final = getattr(response_obj, "status", None) if response_obj is not None else None has_tool_call: Final = any( getattr(item, "type", None) == "function_call" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 4c5e2101fbf..fdd08eaa182 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -183,6 +183,10 @@ async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emit def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): + """Fake-streamed responses arrive as one chunk carrying both the delta and the + finish_reason. The refusal has to be split off and streamed as text, or the + client gets ``stop_reason: refusal`` over an empty content array. + """ chunks = [ _make_chunk( Delta(content=None, refusal="I cannot fulfill this request."), @@ -193,6 +197,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): events = _drain_sync(wrapper) + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." @@ -202,7 +207,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async(): chunks = [ _make_chunk( - Delta(content=None, refusal="I cannot fulfill this request."), + Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."}), finish_reason="stop", ) ] @@ -210,6 +215,7 @@ async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_as events = await _drain_async(wrapper) + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." From b9f5cd60364aae12b654c3e01aa43173e4003e71 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Sat, 5 Sep 2026 23:56:09 -0700 Subject: [PATCH 8/8] ci: run unit tests on Python 3.12 (#39989) --- .github/workflows/_test-unit-base.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 6f6822a975b..75b0f93fd77 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -55,17 +55,14 @@ on: permissions: contents: read +env: + UV_PYTHON: "3.12" + jobs: run: - name: ${{ matrix.python-version == '3.12' && 'Run tests' || format('Run tests (Python {0})', matrix.python-version) }} + name: Run tests runs-on: ubuntu-latest timeout-minutes: ${{ inputs.job-timeout-minutes }} - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - env: - UV_PYTHON: ${{ matrix.python-version }} permissions: contents: read pull-requests: read @@ -88,7 +85,7 @@ jobs: timeout-minutes: 3 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.UV_PYTHON }} - name: Set up uv if: steps.changes.outputs.decision != 'skip' @@ -103,9 +100,9 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ env.UV_CACHE_DIR }} - key: ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}-${{ hashFiles('uv.lock') }} + key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }} restore-keys: | - ${{ runner.os }}-uv-downloads-py${{ matrix.python-version }}- + ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}- - name: Cache the Rust build if: steps.changes.outputs.decision != 'skip' @@ -139,7 +136,7 @@ jobs: WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} DIST: ${{ inputs.dist }} - COVERAGE_CORE: ${{ contains(fromJSON('["3.10", "3.11"]'), matrix.python-version) && 'ctrace' || 'sysmon' }} + COVERAGE_CORE: sysmon run: | if [ "${WORKERS}" = "0" ]; then uv run --no-sync pytest ${TEST_PATH:?} \ @@ -166,7 +163,7 @@ jobs: fi - name: Save coverage report - if: always() && matrix.python-version == '3.12' && steps.changes.outputs.decision != 'skip' + if: always() && steps.changes.outputs.decision != 'skip' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 with: name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}