scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

Merge pull request #24912 from Sameerlite/litellm_fix-refusal-status-streaming

Browse source 
fix(responses): map refusal stop_reason to incomplete status in streaming
This commit is contained in:
Sameer Kankute 2026-04-02 18:30:54 +05:30 • committed by GitHub
commit c2f0788541
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 240 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -123,10 +123,13 @@ class ChunkProcessor:
finish_reason = "stop"
for chunk in chunks:
if "choices" in chunk and len(chunk["choices"]) > 0:
chunk_finish_reason = None
if hasattr(chunk["choices"][0], "finish_reason"):
finish_reason = chunk["choices"][0].finish_reason
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
finish_reason = chunk["choices"][0]["finish_reason"]
chunk_finish_reason = chunk["choices"][0]["finish_reason"]
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
# Initialize the response dictionary
response = ModelResponse(

6
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1134,7 +1134,11 @@ class CustomStreamWrapper:
):
if self.received_finish_reason is not None:
_chunk_has_content = isinstance(chunk, dict) and (
bool(chunk.get("text", "")) or chunk.get("tool_use") is not None
bool(chunk.get("text", ""))
or chunk.get("tool_use") is not None
# Usage-only final chunks are valid and needed to surface
# finish_reason/usage to downstream translators.
or chunk.get("usage") is not None
)
if not _chunk_has_content and (
not isinstance(chunk, dict)

10
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -1016,14 +1016,18 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
):
if src and isinstance(src, dict):
self._merge_provider_specific_fields(src)
# Emit any just-queued output_item event
if self._pending_response_events:
return self._pending_response_events.pop(0)
# Always snapshot before returning any pending events so that
# finish_reason (e.g. content_filter) is captured even when
# _ensure_output_item_for_chunk queues events on the same chunk.
# This mirrors the async path (see __anext__).
self.collected_chat_completion_chunks.append(
self._snapshot_chunk_for_stream_chunk_builder(
cast(ModelResponseStream, chunk)
)
)
# Emit any just-queued output_item event
if self._pending_response_events:
return self._pending_response_events.pop(0)
response_api_chunk = (
self._transform_chat_completion_chunk_to_response_api_chunk(
chunk

10
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -1519,7 +1519,7 @@ class LiteLLMCompletionResponsesConfig:
"""
Map chat completion finish_reason to responses API status.
Chat completion finish_reason values include: "stop", "length", "tool_calls", "content_filter", "function_call"
Chat completion finish_reason values include: "stop", "length", "tool_calls", "content_filter", "function_call", "refusal"
Responses API status values are: "completed", "failed", "in_progress", "cancelled", "queued", "incomplete"
Args:
@ -1534,7 +1534,7 @@ class LiteLLMCompletionResponsesConfig:
# Map finish reasons to status
if finish_reason in ["stop", "tool_calls", "function_call"]:
return "completed"
elif finish_reason in ["length", "content_filter"]:
elif finish_reason in ["length", "content_filter", "refusal"]:
return "incomplete"
else:
# Default to completed for unknown finish reasons
@ -2123,9 +2123,9 @@ class LiteLLMCompletionResponsesConfig:
hasattr(completion_details, "reasoning_tokens")
and completion_details.reasoning_tokens is not None
):
output_details_dict["reasoning_tokens"] = (
completion_details.reasoning_tokens
)
output_details_dict[
"reasoning_tokens"
] = completion_details.reasoning_tokens
else:
output_details_dict["reasoning_tokens"] = 0

94
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -770,7 +770,9 @@ async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging):
from litellm.llms.vertex_ai.common_utils import VertexAIError
async def _raise_bad_request(**kwargs):
raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None)
raise VertexAIError(
status_code=400, message="invalid maxOutputTokens", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -788,7 +790,9 @@ async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging):
@pytest.mark.asyncio
async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(logging_obj: Logging):
async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(
logging_obj: Logging,
):
"""Ensure Vertex 429 rate-limit errors raise MidStreamFallbackError, not RateLimitError.
Regression test for https://github.com/BerriAI/litellm/issues/20870
@ -797,7 +801,9 @@ async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(logging_o
from litellm.llms.vertex_ai.common_utils import VertexAIError
async def _raise_rate_limit(**kwargs):
raise VertexAIError(status_code=429, message="Resource exhausted.", headers=None)
raise VertexAIError(
status_code=429, message="Resource exhausted.", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -825,7 +831,9 @@ def test_sync_streaming_rate_limit_triggers_midstream_fallback(logging_obj: Logg
from litellm.llms.vertex_ai.common_utils import VertexAIError
def _raise_rate_limit(**kwargs):
raise VertexAIError(status_code=429, message="Resource exhausted.", headers=None)
raise VertexAIError(
status_code=429, message="Resource exhausted.", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -850,7 +858,9 @@ def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging):
from litellm.llms.vertex_ai.common_utils import VertexAIError
def _raise_bad_request(**kwargs):
raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None)
raise VertexAIError(
status_code=400, message="invalid maxOutputTokens", headers=None
)
response = CustomStreamWrapper(
completion_stream=None,
@ -1363,6 +1373,7 @@ def _build_chunks(pattern: list[str], N: int) -> list[ModelResponseStream]:
chunks.append(_make_chunk(p))
return chunks
_REPETITION_TEST_CASES = [
# Basic cases
pytest.param(
@ -1419,7 +1430,14 @@ _REPETITION_TEST_CASES = [
id="last_chunk_different_no_raise",
),
pytest.param(
["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + ["different_mid"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1),
["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1)
+ ["different_mid"]
+ ["same"]
* (
litellm.REPEATED_STREAMING_CHUNK_LIMIT
- litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2
+ 1
),
False,
id="middle_chunk_different_no_raise",
),
@ -1429,7 +1447,9 @@ _REPETITION_TEST_CASES = [
id="last_two_different_no_raise",
),
pytest.param(
["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["diff"],
["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT
+ ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT
+ ["diff"],
True,
id="in_between_same_and_diff_raise",
),
@ -1455,6 +1475,8 @@ def test_raise_on_model_repetition(
for chunk in chunks:
wrapper.chunks.append(chunk)
wrapper.raise_on_model_repetition()
def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj):
"""
Test that provider-reported usage from a post-finish_reason chunk
@ -1536,12 +1558,13 @@ def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj):
last_chunk = collected[-1]
hidden_usage = last_chunk._hidden_params.get("usage")
assert hidden_usage is not None, "Expected usage in _hidden_params"
assert hidden_usage.prompt_tokens == 20, (
f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}"
)
assert hidden_usage.completion_tokens == 135, (
f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}"
)
assert (
hidden_usage.prompt_tokens == 20
), f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}"
assert (
hidden_usage.completion_tokens == 135
), f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}"
@pytest.mark.asyncio
async def test_custom_stream_wrapper_aclose():
@ -1615,9 +1638,9 @@ def test_content_not_dropped_when_finish_reason_already_set(
result = initialized_custom_stream_wrapper.chunk_creator(chunk=content_chunk)
assert result is not None, (
"chunk_creator() returned None — content was dropped (issue #22098)"
)
assert (
result is not None
), "chunk_creator() returned None — content was dropped (issue #22098)"
assert result.choices[0].delta.content == "world!"
@ -1669,18 +1692,45 @@ def test_tool_use_not_dropped_when_finish_reason_already_set(
result = initialized_custom_stream_wrapper.chunk_creator(chunk=tool_chunk)
assert result is not None, (
"chunk_creator() returned None — tool_use data was dropped"
)
assert (
result is not None
), "chunk_creator() returned None — tool_use data was dropped"
tool_calls = result.choices[0].delta.tool_calls
assert tool_calls is not None and len(tool_calls) > 0, (
"tool_calls should contain at least one tool call"
)
assert (
tool_calls is not None and len(tool_calls) > 0
), "tool_calls should contain at least one tool call"
assert tool_calls[0].id == "call_1"
assert tool_calls[0].function.name == "get_weather"
def test_usage_only_chunk_not_dropped_when_finish_reason_already_set(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""
Regression test: usage-only chunks must not be dropped once finish_reason
is already set. Dropping these chunks can lose terminal finish_reason in
downstream Responses API streaming translation.
"""
initialized_custom_stream_wrapper.received_finish_reason = "content_filter"
initialized_custom_stream_wrapper.custom_llm_provider = "anthropic"
usage_only_chunk = {
"text": "",
"tool_use": None,
"is_finished": False,
"finish_reason": "",
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
"index": 0,
}
result = initialized_custom_stream_wrapper.chunk_creator(chunk=usage_only_chunk)
assert result is not None, "usage-only chunk should not be dropped"
assert result.choices[0].finish_reason == "content_filter"
assert result.usage is not None
@pytest.mark.asyncio
async def test_custom_stream_wrapper_anext_does_not_block_event_loop_for_sync_iterators(
logging_obj: Logging,

202
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -504,6 +504,35 @@ class TestLiteLLMCompletionResponsesConfig:
]
assert item.status != "stop"
def test_transform_chat_completion_response_status_with_refusal(self):
"""
`finish_reason=refusal` should map to `status=incomplete` in Responses API.
"""
chat_completion_response = ModelResponse(
id="test-response-id",
created=1234567890,
model="claude-sonnet-4-5",
object="chat.completion",
choices=[
Choices(
finish_reason="refusal",
index=0,
message=Message(
content="",
role="assistant",
),
)
],
)
responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
request_input="this is a test",
responses_api_request={},
chat_completion_response=chat_completion_response,
)
assert responses_api_response.status == "incomplete"
def test_transform_chat_completion_response_preserves_hidden_params(self):
"""Test that _hidden_params from chat completion response are preserved in responses API response"""
# Setup
@ -976,10 +1005,11 @@ class TestToolTransformation:
tools = [vertex_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -999,10 +1029,11 @@ class TestToolTransformation:
tools = [mcp_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1022,10 +1053,11 @@ class TestToolTransformation:
tools = [computer_use_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1045,10 +1077,11 @@ class TestToolTransformation:
tools = [web_search_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1077,10 +1110,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1108,10 +1142,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1135,10 +1170,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1162,10 +1198,11 @@ class TestToolTransformation:
tools = [code_execution_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1187,10 +1224,11 @@ class TestToolTransformation:
tools = [tool_search_regex, tool_search_bm25]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1220,10 +1258,11 @@ class TestToolTransformation:
]
# Execute
result_tools, web_search_options = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
web_search_options,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1256,10 +1295,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1280,10 +1320,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1302,10 +1343,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -1325,10 +1367,11 @@ class TestToolTransformation:
tools = [function_tool]
# Execute
result_tools, _ = (
LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
(
result_tools,
_,
) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
tools=tools
)
# Assert
@ -2055,6 +2098,53 @@ class TestEnsureOutputItemContentPartAdded:
assert events[1].part.type == "output_text"
assert iterator.sent_content_part_added_event is True
def test_emit_response_completed_uses_stream_finish_reason(self):
"""
When the assembled model response carries finish_reason="content_filter"
(snapshotted from the underlying stream before any pending events fire),
_emit_response_completed_event must produce status="incomplete".
"""
from unittest.mock import Mock
import litellm
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
mock_stream_wrapper = Mock(spec=litellm.CustomStreamWrapper)
mock_stream_wrapper.logging_obj = Mock()
iterator = LiteLLMCompletionStreamingIterator(
model="anthropic/claude-sonnet-4-6",
litellm_custom_stream_wrapper=mock_stream_wrapper,
request_input="test",
responses_api_request={},
custom_llm_provider="anthropic",
)
litellm_model_response = ModelResponse(
id="chatcmpl-test",
created=1234567890,
model="anthropic/claude-sonnet-4-6",
object="chat.completion",
choices=[
Choices(
finish_reason="content_filter",
index=0,
message=Message(content="", role="assistant"),
)
],
usage=Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11),
)
completed_event = iterator._emit_response_completed_event(
litellm_model_response
)
assert completed_event is not None
assert completed_event.response.status == "incomplete"
assert completed_event.response.output[0].status == "incomplete"
def test_reasoning_item_does_not_emit_content_part_added(self):
"""Reasoning items should not get a content_part.added event."""
from litellm.types.llms.openai import OutputItemAddedEvent