diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 29347f6554a..b500669e818 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -2706,7 +2706,7 @@ class AmazonConverseConfig(BaseConfig): # When json_mode filtered out all synthetic tool calls the response # is plain content, not a pending tool invocation. Fix finish_reason # so callers (e.g. OpenAI SDK) don't misinterpret it. - if resolved_json_mode and not filtered_tools and tools: + if resolved_json_mode and not filtered_tools and tools and initial_finish_reason != "length": initial_finish_reason = "stop" ( diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index c215e3f8395..802fee508d3 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -35,6 +35,7 @@ from litellm.types.llms.openai import ( ReasoningSummaryTextDoneEvent, ResponseCompletedEvent, ResponseCreatedEvent, + ResponseIncompleteEvent, ResponseInProgressEvent, ResponseInputParam, ResponsesAPIOptionalRequestParams, @@ -1266,7 +1267,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) return reasoning_aligned - def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None: + def _emit_response_completed_event( + self, litellm_model_response: ModelResponse + ) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: if litellm_model_response: # Transform the response responses_api_response: Final = ( @@ -1286,6 +1289,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): # Encode the response ID to match non-streaming behavior encoded_response: Final = self._with_encoded_response_id(responses_api_response) + if encoded_response.status == "incomplete": + return ResponseIncompleteEvent( + type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + response=encoded_response, + ) return ResponseCompletedEvent( type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=encoded_response, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index e1c7cd4b890..e57e7fd2ebe 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2417,13 +2417,13 @@ class LiteLLMCompletionResponsesConfig: responses_api_request=responses_api_request, ), parallel_tool_calls=getattr(chat_completion_response, "parallel_tool_calls", False), - temperature=getattr(chat_completion_response, "temperature", 0), + temperature=responses_api_request.get("temperature", 0), tool_choice=LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response( responses_api_request.get("tool_choice") ), tools=getattr(chat_completion_response, "tools", []), - top_p=getattr(chat_completion_response, "top_p", None), - max_output_tokens=getattr(chat_completion_response, "max_output_tokens", None), + top_p=responses_api_request.get("top_p"), + max_output_tokens=responses_api_request.get("max_output_tokens"), previous_response_id=getattr(chat_completion_response, "previous_response_id", None), reasoning=None, status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 5e045c3e84f..a0668de7a5e 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1239,7 +1239,10 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): evt: Final = self._events[self._idx] self._idx += 1 openai_types: Final = _get_openai_response_types() - if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + if getattr(evt, "type", None) in ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ): self.completed_response = evt self._log_completed_response(is_async=True) return evt @@ -1253,7 +1256,10 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): evt: Final = self._events[self._idx] self._idx += 1 openai_types: Final = _get_openai_response_types() - if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + if getattr(evt, "type", None) in ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ): self.completed_response = evt self._log_completed_response(is_async=False) return evt @@ -1306,7 +1312,10 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): evt: Final = self._events[self._idx] self._idx += 1 openai_types: Final = _get_openai_response_types() - if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + if getattr(evt, "type", None) in ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ): self.completed_response = evt self._log_completed_response(is_async=True) return evt @@ -1320,7 +1329,10 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): evt: Final = self._events[self._idx] self._idx += 1 openai_types: Final = _get_openai_response_types() - if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + if getattr(evt, "type", None) in ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ): self.completed_response = evt self._log_completed_response(is_async=False) return evt @@ -1723,6 +1735,14 @@ def build_synthetic_response_events( ) ) + if transformed.status == "incomplete": + events.append( + openai_types.ResponseIncompleteEvent( + type=openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + response=transformed, + ) + ) + return events events.append( openai_types.ResponseCompletedEvent( type=openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index e4a50317190..a131c3fe13c 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -8109,3 +8109,36 @@ def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: p ) assert AmazonConverseConfig._supports_sampling_params("custom-test-reasoning-model") is False assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True + + +@pytest.mark.parametrize( + "stop_reason, expected_finish_reason", + [("max_tokens", "length"), ("end_turn", "stop")], +) +def test_json_mode_filtered_tool_call_keeps_length_finish_reason(stop_reason, expected_finish_reason): + body = { + "output": { + "message": { + "role": "assistant", + "content": [{"toolUse": {"toolUseId": "t1", "name": "json_tool_call", "input": {"a": "x"}}}], + } + }, + "stopReason": stop_reason, + "usage": {"inputTokens": 10, "outputTokens": 60, "totalTokens": 70}, + } + http_response = MagicMock(status_code=200, headers={}, text="") + http_response.json.return_value = body + + out = AmazonConverseConfig()._transform_response( + model="anthropic.claude-sonnet-4-5", + response=http_response, + model_response=ModelResponse(), + stream=False, + logging_obj=MagicMock(), + optional_params={"json_mode": True}, + api_key="", + data={}, + messages=[], + encoding=None, + ) + assert out.choices[0].finish_reason == expected_finish_reason diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 7b9de4644b4..7dcde644cbb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1,6 +1,8 @@ +import asyncio import json from copy import deepcopy from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock import pytest from openai.types.responses.response_function_web_search import ( @@ -13,10 +15,22 @@ from openai.types.responses.response_function_web_search import ( import litellm from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt +from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, +) from litellm.responses.litellm_completion_transformation.transformation import ( TOOL_CALLS_CACHE, LiteLLMCompletionResponsesConfig, ) +from litellm.responses.streaming_iterator import ( + CachedResponsesAPIStreamingIterator, + build_synthetic_response_events, +) +from litellm.types.llms.openai import ( + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) from litellm.types.responses.main import build_web_search_call from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -5172,3 +5186,110 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool ) assert exc_info.value.status_code == 400 assert "tool_choice={'type': 'file_search'}" in str(exc_info.value) + + +def _chat_response(finish_reason: str) -> ModelResponse: + choice = Choices(index=0, finish_reason="stop", message=Message(role="assistant", content="cut off")) + choice.finish_reason = finish_reason + return ModelResponse( + id="chatcmpl-1", + model="some-model", + choices=[choice], + usage=Usage(prompt_tokens=10, completion_tokens=60, total_tokens=70), + ) + + +def _transform(finish_reason: str, request: ResponsesAPIOptionalRequestParams) -> ResponsesAPIResponse: + return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( + request_input="hi", + responses_api_request=request, + chat_completion_response=_chat_response(finish_reason), + ) + + +@pytest.mark.parametrize( + "finish_reason, reason", + [("length", "max_output_tokens"), ("content_filter", "content_filter")], +) +def test_truncated_finish_reason_sets_incomplete_details(finish_reason: str, reason: str) -> None: + result = _transform(finish_reason, {}) + assert result.status == "incomplete" + assert result.incomplete_details is not None + assert result.incomplete_details.reason == reason + + +def test_stop_finish_reason_has_no_incomplete_details() -> None: + result = _transform("stop", {}) + assert result.status == "completed" + assert result.incomplete_details is None + + +def test_request_sampling_params_are_echoed() -> None: + result = _transform("stop", {"temperature": 0.3, "top_p": 0.9, "max_output_tokens": 60}) + assert (result.temperature, result.top_p, result.max_output_tokens) == (0.3, 0.9, 60) + + +@pytest.mark.parametrize( + "finish_reason, event_type", + [ + ("length", ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE), + ("stop", ResponsesAPIStreamEvents.RESPONSE_COMPLETED), + ], +) +def test_stream_terminal_event_follows_status(finish_reason: str, event_type: ResponsesAPIStreamEvents) -> None: + iterator = LiteLLMCompletionStreamingIterator( + model="some-model", + litellm_custom_stream_wrapper=AsyncMock(), + request_input="hi", + responses_api_request={}, + ) + event = iterator._emit_response_completed_event(_chat_response(finish_reason)) + assert event is not None + assert event.type == event_type + + +@pytest.mark.parametrize( + "finish_reason, event_type", + [ + ("length", ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE), + ("stop", ResponsesAPIStreamEvents.RESPONSE_COMPLETED), + ], +) +def test_replayed_stream_terminal_event_follows_status( + finish_reason: str, event_type: ResponsesAPIStreamEvents +) -> None: + events = build_synthetic_response_events( + transformed=_transform(finish_reason, {}), + logging_obj=None, + chunk_size=10, + ) + assert events[-1].type == event_type + + +def test_omitted_temperature_defaults_to_zero() -> None: + assert _transform("stop", {}).temperature == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("finish_reason", ["length", "stop"]) +async def test_replayed_stream_logs_success_exactly_once(finish_reason: str) -> None: + logging_obj = MagicMock() + logging_obj._on_deferred_stream_complete = None + logging_obj.dispatch_success_handlers = AsyncMock() + iterator = CachedResponsesAPIStreamingIterator( + response=_transform(finish_reason, {}), + logging_obj=logging_obj, + ) + async for _ in iterator: + pass + await asyncio.sleep(0) + assert logging_obj.dispatch_success_handlers.await_count == 1 + + +def test_omitted_temperature_is_not_forwarded_to_the_chat_request() -> None: + result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="some-model", + input="hi", + responses_api_request={}, + ) + assert result.get("temperature") is None