mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(responses_bridge): map incomplete responses to finish_reason length instead of 500
This commit is contained in:
mateo-berri
parent
e07a7129c5
commit
abc6ebfb33
2 changed files with 328 additions and 28 deletions
|
|
@ -113,6 +113,48 @@ def _build_reasoning_item(
|
|||
}
|
||||
|
||||
|
||||
def _reasoning_item_from_output_item(item: object) -> _BuiltReasoningItem | None:
|
||||
from openai.types.responses import ResponseReasoningItem
|
||||
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
return _build_reasoning_item(
|
||||
item_id=item.id,
|
||||
encrypted_content=getattr(item, "encrypted_content", None),
|
||||
summary_raw=item.summary,
|
||||
)
|
||||
if isinstance(item, dict) and item.get("type") == "reasoning":
|
||||
return _build_reasoning_item(
|
||||
item_id=item.get("id", ""),
|
||||
encrypted_content=item.get("encrypted_content"),
|
||||
summary_raw=item.get("summary"),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _reasoning_items_from_output_items(output_items: Sequence[object]) -> tuple[_BuiltReasoningItem, ...]:
|
||||
return tuple(
|
||||
reasoning_item
|
||||
for reasoning_item in (_reasoning_item_from_output_item(item) for item in output_items)
|
||||
if reasoning_item is not None
|
||||
)
|
||||
|
||||
|
||||
def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]:
|
||||
if incomplete_reason == "content_filter":
|
||||
return "content_filter"
|
||||
return "length"
|
||||
|
||||
|
||||
def _incomplete_reason_from_response_payload(response_payload: object) -> str | None:
|
||||
if not isinstance(response_payload, Mapping):
|
||||
return None
|
||||
incomplete_details: Final = response_payload.get("incomplete_details")
|
||||
if not isinstance(incomplete_details, Mapping):
|
||||
return None
|
||||
reason: Final = incomplete_details.get("reason")
|
||||
return reason if isinstance(reason, str) else None
|
||||
|
||||
|
||||
class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False):
|
||||
provider_specific_fields: Mapping[str, object]
|
||||
|
||||
|
|
@ -657,6 +699,30 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
return choices
|
||||
|
||||
@staticmethod
|
||||
def _build_empty_incomplete_choice(
|
||||
output_items: Sequence[object],
|
||||
finish_reason: Literal["length", "content_filter"],
|
||||
) -> "Choices":
|
||||
from litellm.types.utils import Choices, Message
|
||||
|
||||
reasoning_items: Final = _reasoning_items_from_output_items(output_items)
|
||||
reasoning_content: Final = " ".join(
|
||||
summary_block["text"]
|
||||
for reasoning_item in reasoning_items
|
||||
for summary_block in reasoning_item["summary"]
|
||||
if summary_block.get("text")
|
||||
)
|
||||
message: Final = Message(
|
||||
content="",
|
||||
reasoning_content=reasoning_content if reasoning_content else None,
|
||||
reasoning_items=cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
reasoning_items or None,
|
||||
),
|
||||
)
|
||||
return Choices(message=message, finish_reason=finish_reason, index=0)
|
||||
|
||||
@classmethod
|
||||
def _extract_output_from_completed_event(cls, parsed_chunk: Mapping[str, object]) -> list[dict[str, object]] | None:
|
||||
response_payload: Final = parsed_chunk.get("response")
|
||||
|
|
@ -763,11 +829,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
handle_raw_dict_callback=self._handle_raw_dict_response_item,
|
||||
)
|
||||
|
||||
if len(choices) == 0:
|
||||
if raw_response.incomplete_details is not None and raw_response.incomplete_details.reason is not None:
|
||||
raise ValueError(f"{model} unable to complete request: {raw_response.incomplete_details.reason}")
|
||||
response_is_incomplete: Final = (
|
||||
raw_response.status == "incomplete" or raw_response.incomplete_details is not None
|
||||
)
|
||||
|
||||
if len(choices) == 0 and not response_is_incomplete:
|
||||
raise ValueError(f"Unknown items in responses API response: {output_items}")
|
||||
|
||||
if response_is_incomplete:
|
||||
incomplete_finish_reason: Final = _map_incomplete_reason_to_finish_reason(
|
||||
raw_response.incomplete_details.reason if raw_response.incomplete_details is not None else None
|
||||
)
|
||||
if len(choices) == 0:
|
||||
choices.append(self._build_empty_incomplete_choice(output_items, incomplete_finish_reason))
|
||||
else:
|
||||
raise ValueError(f"Unknown items in responses API response: {output_items}")
|
||||
for choice in choices:
|
||||
choice.finish_reason = incomplete_finish_reason
|
||||
|
||||
setattr(model_response, "choices", choices)
|
||||
|
||||
|
|
@ -1392,12 +1469,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
]
|
||||
)
|
||||
elif event_type == "response.completed":
|
||||
# Response is fully complete - now we can signal is_finished=True
|
||||
# This ensures we don't prematurely end the stream before tool_calls arrive
|
||||
|
||||
# Check if response contains function_call items in output
|
||||
# to determine correct finish_reason
|
||||
elif event_type in ("response.completed", "response.incomplete"):
|
||||
response_data: Final = parsed_chunk.get("response", {})
|
||||
output_items: Final = response_data.get("output", []) if response_data else []
|
||||
|
||||
|
|
@ -1407,25 +1479,16 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
if isinstance(item, dict)
|
||||
)
|
||||
|
||||
finish_reason: Final = "tool_calls" if has_function_calls else "stop"
|
||||
finish_reason: Final = (
|
||||
_map_incomplete_reason_to_finish_reason(_incomplete_reason_from_response_payload(response_data))
|
||||
if event_type == "response.incomplete"
|
||||
else ("tool_calls" if has_function_calls else "stop")
|
||||
)
|
||||
|
||||
# Extract reasoning items with encrypted_content for round-tripping
|
||||
completed_reasoning_items: list[_BuiltReasoningItem] | None = None
|
||||
for item in output_items:
|
||||
if not isinstance(item, dict) or item.get("type") != "reasoning":
|
||||
continue
|
||||
if completed_reasoning_items is None:
|
||||
completed_reasoning_items = []
|
||||
completed_reasoning_items.append(
|
||||
_build_reasoning_item(
|
||||
item_id=item.get("id", ""),
|
||||
encrypted_content=item.get("encrypted_content"),
|
||||
summary_raw=item.get("summary"),
|
||||
)
|
||||
)
|
||||
completed_reasoning_items_typed: Final = cast(
|
||||
terminal_reasoning_items: Final = _reasoning_items_from_output_items(output_items)
|
||||
terminal_reasoning_items_typed: Final = cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
completed_reasoning_items,
|
||||
list(terminal_reasoning_items) if terminal_reasoning_items else None,
|
||||
)
|
||||
|
||||
usage = None
|
||||
|
|
@ -1439,7 +1502,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
index=0,
|
||||
delta=Delta(
|
||||
content="",
|
||||
reasoning_items=completed_reasoning_items_typed,
|
||||
reasoning_items=terminal_reasoning_items_typed,
|
||||
),
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3485,3 +3485,240 @@ async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire(
|
|||
post_kwargs = mock_post.call_args.kwargs
|
||||
request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"])
|
||||
assert request_body["tool_choice"] == expected_wire_tool_choice
|
||||
|
||||
|
||||
def _make_incomplete_responses_api_response(incomplete_reason, output):
|
||||
from litellm.types.llms.openai import (
|
||||
InputTokensDetails,
|
||||
OutputTokensDetails,
|
||||
ResponseAPIUsage,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_incomplete",
|
||||
created_at=1760144904,
|
||||
error=None,
|
||||
incomplete_details={"reason": incomplete_reason} if incomplete_reason else None,
|
||||
instructions=None,
|
||||
metadata={},
|
||||
model="gpt-5.6-sol",
|
||||
object="response",
|
||||
output=output,
|
||||
parallel_tool_calls=True,
|
||||
temperature=1.0,
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
top_p=1.0,
|
||||
max_output_tokens=16,
|
||||
previous_response_id=None,
|
||||
reasoning={"effort": "high", "summary": None},
|
||||
status="incomplete",
|
||||
text={"format": {"type": "text"}, "verbosity": "medium"},
|
||||
truncation="disabled",
|
||||
usage=ResponseAPIUsage(
|
||||
input_tokens=37,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
audio_tokens=None, cached_tokens=0, text_tokens=None
|
||||
),
|
||||
output_tokens=16,
|
||||
output_tokens_details=OutputTokensDetails(
|
||||
reasoning_tokens=16, text_tokens=None
|
||||
),
|
||||
total_tokens=53,
|
||||
cost=None,
|
||||
),
|
||||
user=None,
|
||||
store=True,
|
||||
background=False,
|
||||
billing={"payer": "developer"},
|
||||
max_tool_calls=None,
|
||||
prompt_cache_key=None,
|
||||
safety_identifier=None,
|
||||
service_tier="default",
|
||||
top_logprobs=0,
|
||||
)
|
||||
|
||||
|
||||
def _make_reasoning_only_output_item():
|
||||
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
|
||||
|
||||
return ResponseReasoningItem(
|
||||
id="rs_incomplete",
|
||||
summary=[],
|
||||
type="reasoning",
|
||||
content=None,
|
||||
encrypted_content="enc_abc",
|
||||
status=None,
|
||||
)
|
||||
|
||||
|
||||
def _call_transform_response(handler, raw_response):
|
||||
logging_obj = Mock()
|
||||
logging_obj.model_call_details = {}
|
||||
return handler.transform_response(
|
||||
model="gpt-5.6-sol",
|
||||
raw_response=raw_response,
|
||||
model_response=_make_empty_model_response(),
|
||||
logging_obj=logging_obj,
|
||||
request_data={"model": "gpt-5.6-sol"},
|
||||
messages=[{"role": "user", "content": "compute something hard"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=Mock(),
|
||||
)
|
||||
|
||||
|
||||
def test_transform_response_incomplete_reasoning_only_returns_empty_length_choice():
|
||||
handler = LiteLLMResponsesTransformationHandler()
|
||||
raw_response = _make_incomplete_responses_api_response(
|
||||
"max_output_tokens", [_make_reasoning_only_output_item()]
|
||||
)
|
||||
|
||||
result = _call_transform_response(handler, raw_response)
|
||||
|
||||
assert len(result.choices) == 1
|
||||
choice = result.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
assert choice.index == 0
|
||||
assert choice.message.role == "assistant"
|
||||
assert choice.message.content == ""
|
||||
assert choice.message.reasoning_items[0]["encrypted_content"] == "enc_abc"
|
||||
assert result.usage.prompt_tokens == 37
|
||||
assert result.usage.completion_tokens == 16
|
||||
assert result.usage.total_tokens == 53
|
||||
assert result.usage.completion_tokens_details.reasoning_tokens == 16
|
||||
|
||||
|
||||
def test_transform_response_incomplete_content_filter_maps_finish_reason():
|
||||
handler = LiteLLMResponsesTransformationHandler()
|
||||
raw_response = _make_incomplete_responses_api_response(
|
||||
"content_filter", [_make_reasoning_only_output_item()]
|
||||
)
|
||||
|
||||
result = _call_transform_response(handler, raw_response)
|
||||
|
||||
assert len(result.choices) == 1
|
||||
assert result.choices[0].finish_reason == "content_filter"
|
||||
assert result.choices[0].message.content == ""
|
||||
|
||||
|
||||
def test_transform_response_zero_choices_not_incomplete_still_raises():
|
||||
handler = LiteLLMResponsesTransformationHandler()
|
||||
raw_response = _make_empty_responses_api_response()
|
||||
|
||||
with pytest.raises(ValueError, match="Unknown items"):
|
||||
_call_transform_response(handler, raw_response)
|
||||
|
||||
|
||||
def test_transform_response_incomplete_partial_text_overrides_finish_reason_to_length():
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
|
||||
handler = LiteLLMResponsesTransformationHandler()
|
||||
output_message = ResponseOutputMessage(
|
||||
id="msg_partial",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[], text="partial answer", type="output_text", logprobs=[]
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
status="incomplete",
|
||||
type="message",
|
||||
)
|
||||
raw_response = _make_incomplete_responses_api_response(
|
||||
"max_output_tokens", [_make_reasoning_only_output_item(), output_message]
|
||||
)
|
||||
|
||||
result = _call_transform_response(handler, raw_response)
|
||||
|
||||
assert len(result.choices) == 1
|
||||
choice = result.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
assert choice.message.content == "partial answer"
|
||||
|
||||
|
||||
def test_response_incomplete_stream_event_emits_length_and_usage():
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
|
||||
iterator = OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response=None, sync_stream=True
|
||||
)
|
||||
|
||||
chunk = {
|
||||
"type": "response.incomplete",
|
||||
"response": {
|
||||
"id": "resp_123",
|
||||
"status": "incomplete",
|
||||
"incomplete_details": {"reason": "max_output_tokens"},
|
||||
"output": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"encrypted_content": "enc_abc",
|
||||
"summary": [],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 37,
|
||||
"output_tokens": 16,
|
||||
"output_tokens_details": {"reasoning_tokens": 16},
|
||||
"total_tokens": 53,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk)
|
||||
|
||||
assert len(result.choices) == 1
|
||||
assert result.choices[0].finish_reason == "length"
|
||||
assert result.choices[0].delta.reasoning_items[0]["encrypted_content"] == "enc_abc"
|
||||
assert result.usage is not None
|
||||
assert result.usage.prompt_tokens == 37
|
||||
assert result.usage.completion_tokens == 16
|
||||
assert result.usage.total_tokens == 53
|
||||
|
||||
|
||||
def test_response_incomplete_stream_event_content_filter_maps_finish_reason():
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
|
||||
iterator = OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response=None, sync_stream=True
|
||||
)
|
||||
|
||||
chunk = {
|
||||
"type": "response.incomplete",
|
||||
"response": {
|
||||
"id": "resp_123",
|
||||
"status": "incomplete",
|
||||
"incomplete_details": {"reason": "content_filter"},
|
||||
"output": [],
|
||||
},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk)
|
||||
|
||||
assert result.choices[0].finish_reason == "content_filter"
|
||||
|
||||
|
||||
def test_response_incomplete_stream_event_without_details_defaults_to_length():
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
|
||||
iterator = OpenAiResponsesToChatCompletionStreamIterator(
|
||||
streaming_response=None, sync_stream=True
|
||||
)
|
||||
|
||||
chunk = {
|
||||
"type": "response.incomplete",
|
||||
"response": {"id": "resp_123", "status": "incomplete", "output": []},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk)
|
||||
|
||||
assert result.choices[0].finish_reason == "length"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue