scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

Merge f02487ef4b into 6532dcb73b

Browse source
This commit is contained in:
Tom Ron 2026-10-04 11:59:03 -04:00 • committed by GitHub
commit fc1813e12a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 191 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -2706,7 +2706,7 @@ class AmazonConverseConfig(BaseConfig):
# When json_mode filtered out all synthetic tool calls the response
# is plain content, not a pending tool invocation. Fix finish_reason
# so callers (e.g. OpenAI SDK) don't misinterpret it.
if resolved_json_mode and not filtered_tools and tools:
if resolved_json_mode and not filtered_tools and tools and initial_finish_reason != "length":
initial_finish_reason = "stop"
(

10
litellm/responses/litellm_completion_transformation/streaming_iterator.py
View file

@ -35,6 +35,7 @@ from litellm.types.llms.openai import (
ReasoningSummaryTextDoneEvent,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseIncompleteEvent,
ResponseInProgressEvent,
ResponseInputParam,
ResponsesAPIOptionalRequestParams,
@ -1266,7 +1267,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
return reasoning_aligned
def _emit_response_completed_event(self, litellm_model_response: ModelResponse) -> ResponseCompletedEvent | None:
def _emit_response_completed_event(
self, litellm_model_response: ModelResponse
) -> ResponseCompletedEvent | ResponseIncompleteEvent | None:
if litellm_model_response:
# Transform the response
responses_api_response: Final = (
@ -1286,6 +1289,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# Encode the response ID to match non-streaming behavior
encoded_response: Final = self._with_encoded_response_id(responses_api_response)
if encoded_response.status == "incomplete":
return ResponseIncompleteEvent(
type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
response=encoded_response,
)
return ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=encoded_response,

6
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -2417,13 +2417,13 @@ class LiteLLMCompletionResponsesConfig:
responses_api_request=responses_api_request,
),
parallel_tool_calls=getattr(chat_completion_response, "parallel_tool_calls", False),
temperature=getattr(chat_completion_response, "temperature", 0),
temperature=responses_api_request.get("temperature", 0),
tool_choice=LiteLLMCompletionResponsesConfig._transform_tool_choice_for_responses_api_response(
responses_api_request.get("tool_choice")
),
tools=getattr(chat_completion_response, "tools", []),
top_p=getattr(chat_completion_response, "top_p", None),
max_output_tokens=getattr(chat_completion_response, "max_output_tokens", None),
top_p=responses_api_request.get("top_p"),
max_output_tokens=responses_api_request.get("max_output_tokens"),
previous_response_id=getattr(chat_completion_response, "previous_response_id", None),
reasoning=None,
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(

28
litellm/responses/streaming_iterator.py
View file

@ -1239,7 +1239,10 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
evt: Final = self._events[self._idx]
self._idx += 1
openai_types: Final = _get_openai_response_types()
if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
if getattr(evt, "type", None) in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
):
self.completed_response = evt
self._log_completed_response(is_async=True)
return evt
@ -1253,7 +1256,10 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
evt: Final = self._events[self._idx]
self._idx += 1
openai_types: Final = _get_openai_response_types()
if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
if getattr(evt, "type", None) in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
):
self.completed_response = evt
self._log_completed_response(is_async=False)
return evt
@ -1306,7 +1312,10 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
evt: Final = self._events[self._idx]
self._idx += 1
openai_types: Final = _get_openai_response_types()
if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
if getattr(evt, "type", None) in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
):
self.completed_response = evt
self._log_completed_response(is_async=True)
return evt
@ -1320,7 +1329,10 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
evt: Final = self._events[self._idx]
self._idx += 1
openai_types: Final = _get_openai_response_types()
if getattr(evt, "type", None) == openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
if getattr(evt, "type", None) in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
):
self.completed_response = evt
self._log_completed_response(is_async=False)
return evt
@ -1723,6 +1735,14 @@ def build_synthetic_response_events(
)
)
if transformed.status == "incomplete":
events.append(
openai_types.ResponseIncompleteEvent(
type=openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
response=transformed,
)
)
return events
events.append(
openai_types.ResponseCompletedEvent(
type=openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,

33
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -8109,3 +8109,36 @@ def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: p
)
assert AmazonConverseConfig._supports_sampling_params("custom-test-reasoning-model") is False
assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True
@pytest.mark.parametrize(
"stop_reason, expected_finish_reason",
[("max_tokens", "length"), ("end_turn", "stop")],
)
def test_json_mode_filtered_tool_call_keeps_length_finish_reason(stop_reason, expected_finish_reason):
body = {
"output": {
"message": {
"role": "assistant",
"content": [{"toolUse": {"toolUseId": "t1", "name": "json_tool_call", "input": {"a": "x"}}}],
}
},
"stopReason": stop_reason,
"usage": {"inputTokens": 10, "outputTokens": 60, "totalTokens": 70},
}
http_response = MagicMock(status_code=200, headers={}, text="")
http_response.json.return_value = body
out = AmazonConverseConfig()._transform_response(
model="anthropic.claude-sonnet-4-5",
response=http_response,
model_response=ModelResponse(),
stream=False,
logging_obj=MagicMock(),
optional_params={"json_mode": True},
api_key="",
data={},
messages=[],
encoding=None,
)
assert out.choices[0].finish_reason == expected_finish_reason

121
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -1,6 +1,8 @@
import asyncio
import json
from copy import deepcopy
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai.types.responses.response_function_web_search import (
@ -13,10 +15,22 @@ from openai.types.responses.response_function_web_search import (
import litellm
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
LiteLLMCompletionStreamingIterator,
)
from litellm.responses.litellm_completion_transformation.transformation import (
TOOL_CALLS_CACHE,
LiteLLMCompletionResponsesConfig,
)
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
build_synthetic_response_events,
)
from litellm.types.llms.openai import (
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.responses.main import build_web_search_call
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -5172,3 +5186,110 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool
)
assert exc_info.value.status_code == 400
assert "tool_choice={'type': 'file_search'}" in str(exc_info.value)
def _chat_response(finish_reason: str) -> ModelResponse:
choice = Choices(index=0, finish_reason="stop", message=Message(role="assistant", content="cut off"))
choice.finish_reason = finish_reason
return ModelResponse(
id="chatcmpl-1",
model="some-model",
choices=[choice],
usage=Usage(prompt_tokens=10, completion_tokens=60, total_tokens=70),
)
def _transform(finish_reason: str, request: ResponsesAPIOptionalRequestParams) -> ResponsesAPIResponse:
return LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
request_input="hi",
responses_api_request=request,
chat_completion_response=_chat_response(finish_reason),
)
@pytest.mark.parametrize(
"finish_reason, reason",
[("length", "max_output_tokens"), ("content_filter", "content_filter")],
)
def test_truncated_finish_reason_sets_incomplete_details(finish_reason: str, reason: str) -> None:
result = _transform(finish_reason, {})
assert result.status == "incomplete"
assert result.incomplete_details is not None
assert result.incomplete_details.reason == reason
def test_stop_finish_reason_has_no_incomplete_details() -> None:
result = _transform("stop", {})
assert result.status == "completed"
assert result.incomplete_details is None
def test_request_sampling_params_are_echoed() -> None:
result = _transform("stop", {"temperature": 0.3, "top_p": 0.9, "max_output_tokens": 60})
assert (result.temperature, result.top_p, result.max_output_tokens) == (0.3, 0.9, 60)
@pytest.mark.parametrize(
"finish_reason, event_type",
[
("length", ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE),
("stop", ResponsesAPIStreamEvents.RESPONSE_COMPLETED),
],
)
def test_stream_terminal_event_follows_status(finish_reason: str, event_type: ResponsesAPIStreamEvents) -> None:
iterator = LiteLLMCompletionStreamingIterator(
model="some-model",
litellm_custom_stream_wrapper=AsyncMock(),
request_input="hi",
responses_api_request={},
)
event = iterator._emit_response_completed_event(_chat_response(finish_reason))
assert event is not None
assert event.type == event_type
@pytest.mark.parametrize(
"finish_reason, event_type",
[
("length", ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE),
("stop", ResponsesAPIStreamEvents.RESPONSE_COMPLETED),
],
)
def test_replayed_stream_terminal_event_follows_status(
finish_reason: str, event_type: ResponsesAPIStreamEvents
) -> None:
events = build_synthetic_response_events(
transformed=_transform(finish_reason, {}),
logging_obj=None,
chunk_size=10,
)
assert events[-1].type == event_type
def test_omitted_temperature_defaults_to_zero() -> None:
assert _transform("stop", {}).temperature == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("finish_reason", ["length", "stop"])
async def test_replayed_stream_logs_success_exactly_once(finish_reason: str) -> None:
logging_obj = MagicMock()
logging_obj._on_deferred_stream_complete = None
logging_obj.dispatch_success_handlers = AsyncMock()
iterator = CachedResponsesAPIStreamingIterator(
response=_transform(finish_reason, {}),
logging_obj=logging_obj,
)
async for _ in iterator:
pass
await asyncio.sleep(0)
assert logging_obj.dispatch_success_handlers.await_count == 1
def test_omitted_temperature_is_not_forwarded_to_the_chat_request() -> None:
result = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model="some-model",
input="hi",
responses_api_request={},
)
assert result.get("temperature") is None