mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(anthropic): split refusal off a combined finish_reason chunk
A fake-streamed provider hands the adapter one chunk carrying both the delta payload and the finish_reason, which is exactly what the combined chunk splitter exists for, but its content check never listed the refusal. The translation short-circuits on finish_reason, so that refusal text was dropped and the client got `stop_reason: refusal` over an empty content array, the symptom this PR set out to fix. Both refusal accumulators also drop their `mutable-ok` lists for a plain string attribute
This commit is contained in:
parent
c09d34fc4b
commit
05cba21763
3 changed files with 22 additions and 11 deletions
|
|
@ -101,6 +101,10 @@ class _CombinedChunkSplitter:
|
|||
@staticmethod
|
||||
def _is_combined(chunk: "ModelResponseStream") -> bool:
|
||||
"""True if ``chunk`` carries response content AND a finish_reason."""
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
|
||||
choices: Final = _optional_attr_sequence(chunk, "choices")
|
||||
if not choices:
|
||||
return False
|
||||
|
|
@ -115,6 +119,7 @@ class _CombinedChunkSplitter:
|
|||
or _optional_attr(delta, "tool_calls")
|
||||
or _optional_attr(delta, "reasoning_content")
|
||||
or _optional_attr(delta, "thinking_blocks")
|
||||
or openai_chat_refusal_text(delta)
|
||||
)
|
||||
|
||||
_PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
|
||||
|
|
@ -306,7 +311,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
# Synthesized compaction block from compact_20260112 polyfill (streaming).
|
||||
self.compaction_block = compaction_block
|
||||
self.iterations_usage = iterations_usage
|
||||
self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks
|
||||
self._refusal_text: str = ""
|
||||
self.sent_compaction_block: bool = False
|
||||
# Per-phase flags so the compaction block's start/delta/stop events
|
||||
# are emitted (and the public state machine is advanced) in
|
||||
|
|
@ -1001,7 +1006,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
self,
|
||||
processed_chunk: ContentBlockDelta | MessageBlockDelta,
|
||||
) -> ContentBlockDelta | MessageBlockDelta:
|
||||
if processed_chunk.get("type") != "message_delta" or not self._refusal_text_parts:
|
||||
if processed_chunk.get("type") != "message_delta" or not self._refusal_text:
|
||||
return processed_chunk
|
||||
delta: Final = cast(Mapping[str, object], processed_chunk["delta"]) # cast-ok: keys checked before use
|
||||
if delta.get("stop_reason") == "max_tokens":
|
||||
|
|
@ -1017,7 +1022,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
"delta": { # mutable-ok: fresh message_delta payload; never mutated after construction
|
||||
**delta,
|
||||
"stop_reason": "refusal",
|
||||
"stop_details": refusal_stop_details("".join(self._refusal_text_parts)),
|
||||
"stop_details": refusal_stop_details(self._refusal_text),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -1107,13 +1112,13 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
|
||||
if refusal_text is not None:
|
||||
self._refusal_text_parts.append(refusal_text)
|
||||
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
return False
|
||||
|
||||
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
|
||||
if refusal_text is not None:
|
||||
self._refusal_text = self._refusal_text + refusal_text
|
||||
|
||||
(
|
||||
block_type,
|
||||
content_block_start,
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
self._sent_message_start = False
|
||||
self._sent_message_stop = False
|
||||
self._chunk_queue: deque[dict[str, object]] = deque()
|
||||
self._refusal_text_parts: list[str] = [] # mutable-ok: accumulates streamed refusal delta text across chunks
|
||||
self._refusal_text: str = ""
|
||||
self._sync_responses_iterator: Iterator[object] | None = None
|
||||
|
||||
def _make_message_start(self) -> dict[str, object]:
|
||||
|
|
@ -142,7 +142,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "")
|
||||
if not isinstance(delta, str) or not delta:
|
||||
return
|
||||
self._refusal_text_parts.append(delta)
|
||||
self._refusal_text = self._refusal_text + delta
|
||||
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
|
||||
block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
|
||||
if block_idx < 0:
|
||||
|
|
@ -241,7 +241,7 @@ class AnthropicResponsesStreamWrapper:
|
|||
event.get("response") if isinstance(event, dict) else None
|
||||
)
|
||||
output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
|
||||
refusal_text: Final = responses_output_refusal_text(output) or ("".join(self._refusal_text_parts) or None)
|
||||
refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None)
|
||||
status: Final = getattr(response_obj, "status", None) if response_obj is not None else None
|
||||
has_tool_call: Final = any(
|
||||
getattr(item, "type", None) == "function_call"
|
||||
|
|
|
|||
|
|
@ -183,6 +183,10 @@ async def test_streaming_chat_refusal_parked_in_provider_specific_fields_is_emit
|
|||
|
||||
|
||||
def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved():
|
||||
"""Fake-streamed responses arrive as one chunk carrying both the delta and the
|
||||
finish_reason. The refusal has to be split off and streamed as text, or the
|
||||
client gets ``stop_reason: refusal`` over an empty content array.
|
||||
"""
|
||||
chunks = [
|
||||
_make_chunk(
|
||||
Delta(content=None, refusal="I cannot fulfill this request."),
|
||||
|
|
@ -193,6 +197,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved():
|
|||
|
||||
events = _drain_sync(wrapper)
|
||||
|
||||
assert _text_deltas(events) == ["I cannot fulfill this request."]
|
||||
message_delta = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["delta"]["stop_reason"] == "refusal"
|
||||
assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
|
||||
|
|
@ -202,7 +207,7 @@ def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved():
|
|||
async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_async():
|
||||
chunks = [
|
||||
_make_chunk(
|
||||
Delta(content=None, refusal="I cannot fulfill this request."),
|
||||
Delta(content=None, provider_specific_fields={"refusal": "I cannot fulfill this request."}),
|
||||
finish_reason="stop",
|
||||
)
|
||||
]
|
||||
|
|
@ -210,6 +215,7 @@ async def test_streaming_chat_combined_refusal_and_finish_reason_is_preserved_as
|
|||
|
||||
events = await _drain_async(wrapper)
|
||||
|
||||
assert _text_deltas(events) == ["I cannot fulfill this request."]
|
||||
message_delta = next(event for event in events if event["type"] == "message_delta")
|
||||
assert message_delta["delta"]["stop_reason"] == "refusal"
|
||||
assert message_delta["delta"]["stop_details"]["explanation"] == "I cannot fulfill this request."
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue