diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 1935372e5df..f909111a05c 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -123,10 +123,13 @@ class ChunkProcessor: finish_reason = "stop" for chunk in chunks: if "choices" in chunk and len(chunk["choices"]) > 0: + chunk_finish_reason = None if hasattr(chunk["choices"][0], "finish_reason"): - finish_reason = chunk["choices"][0].finish_reason + chunk_finish_reason = chunk["choices"][0].finish_reason elif "finish_reason" in chunk["choices"][0]: - finish_reason = chunk["choices"][0]["finish_reason"] + chunk_finish_reason = chunk["choices"][0]["finish_reason"] + if chunk_finish_reason is not None: + finish_reason = chunk_finish_reason # Initialize the response dictionary response = ModelResponse( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index e402023d240..5442567b1a5 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1134,7 +1134,11 @@ class CustomStreamWrapper: ): if self.received_finish_reason is not None: _chunk_has_content = isinstance(chunk, dict) and ( - bool(chunk.get("text", "")) or chunk.get("tool_use") is not None + bool(chunk.get("text", "")) + or chunk.get("tool_use") is not None + # Usage-only final chunks are valid and needed to surface + # finish_reason/usage to downstream translators. + or chunk.get("usage") is not None ) if not _chunk_has_content and ( not isinstance(chunk, dict) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 8e75ffdff61..767281d43ab 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1016,14 +1016,18 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ): if src and isinstance(src, dict): self._merge_provider_specific_fields(src) - # Emit any just-queued output_item event - if self._pending_response_events: - return self._pending_response_events.pop(0) + # Always snapshot before returning any pending events so that + # finish_reason (e.g. content_filter) is captured even when + # _ensure_output_item_for_chunk queues events on the same chunk. + # This mirrors the async path (see __anext__). self.collected_chat_completion_chunks.append( self._snapshot_chunk_for_stream_chunk_builder( cast(ModelResponseStream, chunk) ) ) + # Emit any just-queued output_item event + if self._pending_response_events: + return self._pending_response_events.pop(0) response_api_chunk = ( self._transform_chat_completion_chunk_to_response_api_chunk( chunk diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 9075373f1cf..8449620c693 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1519,7 +1519,7 @@ class LiteLLMCompletionResponsesConfig: """ Map chat completion finish_reason to responses API status. - Chat completion finish_reason values include: "stop", "length", "tool_calls", "content_filter", "function_call" + Chat completion finish_reason values include: "stop", "length", "tool_calls", "content_filter", "function_call", "refusal" Responses API status values are: "completed", "failed", "in_progress", "cancelled", "queued", "incomplete" Args: @@ -1534,7 +1534,7 @@ class LiteLLMCompletionResponsesConfig: # Map finish reasons to status if finish_reason in ["stop", "tool_calls", "function_call"]: return "completed" - elif finish_reason in ["length", "content_filter"]: + elif finish_reason in ["length", "content_filter", "refusal"]: return "incomplete" else: # Default to completed for unknown finish reasons @@ -2123,9 +2123,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict["reasoning_tokens"] = ( - completion_details.reasoning_tokens - ) + output_details_dict[ + "reasoning_tokens" + ] = completion_details.reasoning_tokens else: output_details_dict["reasoning_tokens"] = 0 diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index aad3de306c7..904493e02d2 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -770,7 +770,9 @@ async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging): from litellm.llms.vertex_ai.common_utils import VertexAIError async def _raise_bad_request(**kwargs): - raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None) + raise VertexAIError( + status_code=400, message="invalid maxOutputTokens", headers=None + ) response = CustomStreamWrapper( completion_stream=None, @@ -788,7 +790,9 @@ async def test_vertex_streaming_bad_request_not_midstream(logging_obj: Logging): @pytest.mark.asyncio -async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(logging_obj: Logging): +async def test_vertex_streaming_rate_limit_triggers_midstream_fallback( + logging_obj: Logging, +): """Ensure Vertex 429 rate-limit errors raise MidStreamFallbackError, not RateLimitError. Regression test for https://github.com/BerriAI/litellm/issues/20870 @@ -797,7 +801,9 @@ async def test_vertex_streaming_rate_limit_triggers_midstream_fallback(logging_o from litellm.llms.vertex_ai.common_utils import VertexAIError async def _raise_rate_limit(**kwargs): - raise VertexAIError(status_code=429, message="Resource exhausted.", headers=None) + raise VertexAIError( + status_code=429, message="Resource exhausted.", headers=None + ) response = CustomStreamWrapper( completion_stream=None, @@ -825,7 +831,9 @@ def test_sync_streaming_rate_limit_triggers_midstream_fallback(logging_obj: Logg from litellm.llms.vertex_ai.common_utils import VertexAIError def _raise_rate_limit(**kwargs): - raise VertexAIError(status_code=429, message="Resource exhausted.", headers=None) + raise VertexAIError( + status_code=429, message="Resource exhausted.", headers=None + ) response = CustomStreamWrapper( completion_stream=None, @@ -850,7 +858,9 @@ def test_sync_streaming_bad_request_not_midstream(logging_obj: Logging): from litellm.llms.vertex_ai.common_utils import VertexAIError def _raise_bad_request(**kwargs): - raise VertexAIError(status_code=400, message="invalid maxOutputTokens", headers=None) + raise VertexAIError( + status_code=400, message="invalid maxOutputTokens", headers=None + ) response = CustomStreamWrapper( completion_stream=None, @@ -1363,6 +1373,7 @@ def _build_chunks(pattern: list[str], N: int) -> list[ModelResponseStream]: chunks.append(_make_chunk(p)) return chunks + _REPETITION_TEST_CASES = [ # Basic cases pytest.param( @@ -1419,7 +1430,14 @@ _REPETITION_TEST_CASES = [ id="last_chunk_different_no_raise", ), pytest.param( - ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + ["different_mid"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1), + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + + ["different_mid"] + + ["same"] + * ( + litellm.REPEATED_STREAMING_CHUNK_LIMIT + - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + + 1 + ), False, id="middle_chunk_different_no_raise", ), @@ -1429,7 +1447,9 @@ _REPETITION_TEST_CASES = [ id="last_two_different_no_raise", ), pytest.param( - ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["diff"], + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + + ["diff"], True, id="in_between_same_and_diff_raise", ), @@ -1455,6 +1475,8 @@ def test_raise_on_model_repetition( for chunk in chunks: wrapper.chunks.append(chunk) wrapper.raise_on_model_repetition() + + def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): """ Test that provider-reported usage from a post-finish_reason chunk @@ -1536,12 +1558,13 @@ def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): last_chunk = collected[-1] hidden_usage = last_chunk._hidden_params.get("usage") assert hidden_usage is not None, "Expected usage in _hidden_params" - assert hidden_usage.prompt_tokens == 20, ( - f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}" - ) - assert hidden_usage.completion_tokens == 135, ( - f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}" - ) + assert ( + hidden_usage.prompt_tokens == 20 + ), f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}" + assert ( + hidden_usage.completion_tokens == 135 + ), f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}" + @pytest.mark.asyncio async def test_custom_stream_wrapper_aclose(): @@ -1615,9 +1638,9 @@ def test_content_not_dropped_when_finish_reason_already_set( result = initialized_custom_stream_wrapper.chunk_creator(chunk=content_chunk) - assert result is not None, ( - "chunk_creator() returned None — content was dropped (issue #22098)" - ) + assert ( + result is not None + ), "chunk_creator() returned None — content was dropped (issue #22098)" assert result.choices[0].delta.content == "world!" @@ -1669,18 +1692,45 @@ def test_tool_use_not_dropped_when_finish_reason_already_set( result = initialized_custom_stream_wrapper.chunk_creator(chunk=tool_chunk) - assert result is not None, ( - "chunk_creator() returned None — tool_use data was dropped" - ) + assert ( + result is not None + ), "chunk_creator() returned None — tool_use data was dropped" tool_calls = result.choices[0].delta.tool_calls - assert tool_calls is not None and len(tool_calls) > 0, ( - "tool_calls should contain at least one tool call" - ) + assert ( + tool_calls is not None and len(tool_calls) > 0 + ), "tool_calls should contain at least one tool call" assert tool_calls[0].id == "call_1" assert tool_calls[0].function.name == "get_weather" +def test_usage_only_chunk_not_dropped_when_finish_reason_already_set( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + Regression test: usage-only chunks must not be dropped once finish_reason + is already set. Dropping these chunks can lose terminal finish_reason in + downstream Responses API streaming translation. + """ + initialized_custom_stream_wrapper.received_finish_reason = "content_filter" + initialized_custom_stream_wrapper.custom_llm_provider = "anthropic" + + usage_only_chunk = { + "text": "", + "tool_use": None, + "is_finished": False, + "finish_reason": "", + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + "index": 0, + } + + result = initialized_custom_stream_wrapper.chunk_creator(chunk=usage_only_chunk) + + assert result is not None, "usage-only chunk should not be dropped" + assert result.choices[0].finish_reason == "content_filter" + assert result.usage is not None + + @pytest.mark.asyncio async def test_custom_stream_wrapper_anext_does_not_block_event_loop_for_sync_iterators( logging_obj: Logging, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 4e44ef9e50c..f53e0391be0 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -504,6 +504,35 @@ class TestLiteLLMCompletionResponsesConfig: ] assert item.status != "stop" + def test_transform_chat_completion_response_status_with_refusal(self): + """ + `finish_reason=refusal` should map to `status=incomplete` in Responses API. + """ + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="claude-sonnet-4-5", + object="chat.completion", + choices=[ + Choices( + finish_reason="refusal", + index=0, + message=Message( + content="", + role="assistant", + ), + ) + ], + ) + + responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="this is a test", + responses_api_request={}, + chat_completion_response=chat_completion_response, + ) + + assert responses_api_response.status == "incomplete" + def test_transform_chat_completion_response_preserves_hidden_params(self): """Test that _hidden_params from chat completion response are preserved in responses API response""" # Setup @@ -976,10 +1005,11 @@ class TestToolTransformation: tools = [vertex_tool] # Execute - result_tools, web_search_options = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -999,10 +1029,11 @@ class TestToolTransformation: tools = [mcp_tool] # Execute - result_tools, web_search_options = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1022,10 +1053,11 @@ class TestToolTransformation: tools = [computer_use_tool] # Execute - result_tools, web_search_options = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1045,10 +1077,11 @@ class TestToolTransformation: tools = [web_search_tool] # Execute - result_tools, web_search_options = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1077,10 +1110,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, web_search_options = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1108,10 +1142,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1135,10 +1170,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1162,10 +1198,11 @@ class TestToolTransformation: tools = [code_execution_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1187,10 +1224,11 @@ class TestToolTransformation: tools = [tool_search_regex, tool_search_bm25] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1220,10 +1258,11 @@ class TestToolTransformation: ] # Execute - result_tools, web_search_options = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + web_search_options, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1256,10 +1295,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1280,10 +1320,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1302,10 +1343,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -1325,10 +1367,11 @@ class TestToolTransformation: tools = [function_tool] # Execute - result_tools, _ = ( - LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( - tools=tools - ) + ( + result_tools, + _, + ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + tools=tools ) # Assert @@ -2055,6 +2098,53 @@ class TestEnsureOutputItemContentPartAdded: assert events[1].part.type == "output_text" assert iterator.sent_content_part_added_event is True + def test_emit_response_completed_uses_stream_finish_reason(self): + """ + When the assembled model response carries finish_reason="content_filter" + (snapshotted from the underlying stream before any pending events fire), + _emit_response_completed_event must produce status="incomplete". + """ + from unittest.mock import Mock + + import litellm + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + + mock_stream_wrapper = Mock(spec=litellm.CustomStreamWrapper) + mock_stream_wrapper.logging_obj = Mock() + + iterator = LiteLLMCompletionStreamingIterator( + model="anthropic/claude-sonnet-4-6", + litellm_custom_stream_wrapper=mock_stream_wrapper, + request_input="test", + responses_api_request={}, + custom_llm_provider="anthropic", + ) + + litellm_model_response = ModelResponse( + id="chatcmpl-test", + created=1234567890, + model="anthropic/claude-sonnet-4-6", + object="chat.completion", + choices=[ + Choices( + finish_reason="content_filter", + index=0, + message=Message(content="", role="assistant"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11), + ) + + completed_event = iterator._emit_response_completed_event( + litellm_model_response + ) + + assert completed_event is not None + assert completed_event.response.status == "incomplete" + assert completed_event.response.output[0].status == "incomplete" + def test_reasoning_item_does_not_emit_content_part_added(self): """Reasoning items should not get a content_part.added event.""" from litellm.types.llms.openai import OutputItemAddedEvent