diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index b2513a5c046..9158ff4569f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -101,6 +101,10 @@ class _CombinedChunkSplitter: @staticmethod def _is_combined(chunk: "ModelResponseStream") -> bool: """True if ``chunk`` carries response content AND a finish_reason.""" + from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( + openai_chat_refusal_text, + ) + choices: Final = _optional_attr_sequence(chunk, "choices") if not choices: return False @@ -115,6 +119,7 @@ class _CombinedChunkSplitter: or _optional_attr(delta, "tool_calls") or _optional_attr(delta, "reasoning_content") or _optional_attr(delta, "thinking_blocks") + or openai_chat_refusal_text(delta) ) _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = ( @@ -306,7 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Synthesized compaction block from compact_20260112 polyfill (streaming). self.compaction_block = compaction_block self.iterations_usage = iterations_usage - self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks + self._refusal_text: str = "" self.sent_compaction_block: bool = False # Per-phase flags so the compaction block's start/delta/stop events # are emitted (and the public state machine is advanced) in @@ -1001,7 +1006,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): self, processed_chunk: ContentBlockDelta | MessageBlockDelta, ) -> ContentBlockDelta | MessageBlockDelta: - if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts: + if processed_chunk.get("type") != "message_delta" or not self._refusal_text: return processed_chunk delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use if delta.get("stop_reason") == "max_tokens": @@ -1017,7 +1022,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "delta": { # mutable-ok: fresh message_delta payload; never mutated after construction **delta, "stop_reason": "refusal", - "stop_details": refusal_stop_details("".join(self._refusal_text_parts)), + "stop_details": refusal_stop_details(self._refusal_text), }, }, ) @@ -1107,13 +1112,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): from .transformation import LiteLLMAnthropicMessagesAdapter - refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) - if refusal_text is not None: - self._refusal_text_parts.append(refusal_text) - if chunk.choices[0].finish_reason is not None: return False + refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta) + if refusal_text is not None: + self._refusal_text = self._refusal_text + refusal_text + ( block_type, content_block_start, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index bcae732ad42..2e0a6a9df8f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -54,7 +54,7 @@ class AnthropicResponsesStreamWrapper: self._sent_message_start = False self._sent_message_stop = False self._chunk_queue: deque[dict[str, object]] = deque() - self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks + self._refusal_text: str = "" self._sync_responses_iterator: Iterator[object] | None = None def _make_message_start(self) -> dict[str, object]: @@ -142,7 +142,7 @@ class AnthropicResponsesStreamWrapper: delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") if not isinstance(delta, str) or not delta: return - self._refusal_text_parts.append(delta) + self._refusal_text = self._refusal_text + delta item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: @@ -241,7 +241,7 @@ class AnthropicResponsesStreamWrapper: event.get("response") if isinstance(event, dict) else None ) output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else () - refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None) + refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None) status: Final = getattr(response_obj, "status", None) if response_obj is not None else None has_tool_call: Final = any( getattr(item, "type", None) == "function_call" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py index 4c5e2101fbf..fdd08eaa182 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_first_delta.py @@ -183,6 +183,10 @@ async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emit def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): + """Fake-streamed responses arrive as one chunk carrying both the delta and the + finish_reason. The refusal has to be split off and streamed as text, or the + client gets ``stop_reason: refusal`` over an empty content array. + """ chunks = [ _make_chunk( Delta(content=None, refusal="I cannot fulfill this request."), @@ -193,6 +197,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): events = _drain_sync(wrapper) + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request." @@ -202,7 +207,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved(): async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async(): chunks = [ _make_chunk( - Delta(content=None, refusal="I cannot fulfill this request."), + Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."}), finish_reason="stop", ) ] @@ -210,6 +215,7 @@ async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_as events = await _drain_async(wrapper) + assert _text_deltas(events) == ["I cannot fulfill this request."] message_delta = next(event for event in events if event["type"] == "message_delta") assert message_delta["delta"]["stop_reason"] == "refusal" assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."