diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 15f5b28e30e..6b95bb2d9e7 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Coroutine, Optional, Union from typing_extensions import TypedDict +from litellm.responses.response_construction import construct_responses_api_response from litellm.types.llms.openai import ResponsesAPIResponse if TYPE_CHECKING: @@ -51,10 +52,7 @@ class ResponsesToCompletionBridgeHandler: if isinstance(response_obj, ResponsesAPIResponse): response = response_obj elif isinstance(response_obj, dict): - try: - response = ResponsesAPIResponse(**response_obj) - except Exception: - response = ResponsesAPIResponse.model_construct(**response_obj) + response = construct_responses_api_response(response_obj) else: raise ValueError("Unexpected responses stream payload") diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 8b5fae4ef35..d87b6bd986f 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -7,6 +7,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo ) from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.response_construction import construct_responses_api_response from litellm.responses.sse_output_recovery import ( parse_sse_json_chunk, record_output_item_chunk, @@ -206,10 +207,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): response_payload["output"] = [item for _, item in sorted(streamed_output_items.items())] if "created_at" in response_payload: response_payload["created_at"] = _safe_convert_created_field(response_payload["created_at"]) - try: - return ResponsesAPIResponse(**response_payload) - except Exception: - return ResponsesAPIResponse.model_construct(**response_payload) + return construct_responses_api_response(response_payload) def _extract_error_message(self, parsed_chunk: Dict[str, Any]) -> Optional[str]: error_obj = parsed_chunk.get("error") or (parsed_chunk.get("response") or {}).get("error") diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 0db53f90330..5cc95ff4fef 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -12,6 +12,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo ) from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.response_construction import construct_responses_api_response from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( ResponseAPIUsage, @@ -216,11 +217,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): # This allows the response object to be created even when the API doesn't return an id raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" - try: - response = ResponsesAPIResponse(**raw_response_json) - except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") - response = ResponsesAPIResponse.model_construct(**raw_response_json) + response = construct_responses_api_response(raw_response_json) # Store processed headers in additional_headers so they get returned to the client response._hidden_params["additional_headers"] = processed_headers @@ -304,11 +301,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): # Generate a placeholder id for failed responses raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" - try: - response = ResponsesAPIResponse(**raw_response_json) - except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") - response = ResponsesAPIResponse.model_construct(**raw_response_json) + response = construct_responses_api_response(raw_response_json) # Store processed headers in additional_headers so they get returned to the client response._hidden_params["additional_headers"] = processed_headers diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 3c2ae238a0b..815f20b85bc 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -12,6 +12,10 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo _safe_convert_created_field, ) from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.responses.response_construction import ( + construct_responses_api_response, + construct_responses_api_stream_event, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import * from litellm.types.responses.main import * @@ -279,11 +283,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) raw_response_headers = dict(raw_response.headers) processed_headers = process_response_headers(raw_response_headers) - try: - response = ResponsesAPIResponse(**raw_response_json) - except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") - response = ResponsesAPIResponse.model_construct(**raw_response_json) + response = construct_responses_api_response(raw_response_json) # Store processed headers in additional_headers so they get returned to the client response._hidden_params["additional_headers"] = processed_headers @@ -349,7 +349,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): event_pydantic_model.__name__, parsed_chunk, ) - return event_pydantic_model.model_construct(**parsed_chunk) + return construct_responses_api_stream_event(event_pydantic_model, parsed_chunk) @staticmethod def get_event_model_class(event_type: str) -> Any: @@ -646,11 +646,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): raw_response_headers = dict(raw_response.headers) processed_headers = process_response_headers(raw_response_headers) - try: - response = ResponsesAPIResponse(**raw_response_json) - except Exception: - verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") - response = ResponsesAPIResponse.model_construct(**raw_response_json) + response = construct_responses_api_response(raw_response_json) response._hidden_params["additional_headers"] = processed_headers response._hidden_params["headers"] = raw_response_headers diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 4b20962e100..9eaab68cccf 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.response_construction import construct_responses_api_response from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( ResponseInputParam, @@ -259,12 +260,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_headers = dict(raw_response.headers) processed_headers = process_response_headers(raw_response_headers) - try: - response = ResponsesAPIResponse.model_validate(raw_response_json) - except Exception: - verbose_logger.debug("Volcengine Responses API: falling back to model_construct for response parsing.") - construct_response: Callable[..., ResponsesAPIResponse] = ResponsesAPIResponse.model_construct - response = construct_response(**raw_response_json) + response = construct_responses_api_response(raw_response_json) response._hidden_params["additional_headers"] = processed_headers response._hidden_params["headers"] = raw_response_headers diff --git a/litellm/responses/response_construction.py b/litellm/responses/response_construction.py new file mode 100644 index 00000000000..d1487c5b7bc --- /dev/null +++ b/litellm/responses/response_construction.py @@ -0,0 +1,109 @@ +""" +Best-effort construction of Responses API objects from raw provider payloads. + +Providers deviate from the OpenAI Responses spec often enough that strict +pydantic validation fails (missing ``output``, a ``usage`` block with unexpected +fields, ...). Callers used to fall back to ``model_construct``, which skips +validation *and* nested model coercion, so ``response.usage`` stayed a plain +dict and ``response.output`` could be missing entirely. Everything downstream +reads those attributes as declared (``ResponsesAPIResponse.usage`` is a +``ResponseAPIUsage``, ``.output`` is a list), so the fallback produced objects +that violate their own type and blew up with ``AttributeError`` mid-stream. + +These helpers keep the declared shape intact even when validation fails, so the +fallback stays a degraded-data path instead of a crash path. +""" + +from collections.abc import Callable, Mapping + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_logger +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse + +_FIELD_MAPPING_ADAPTER = TypeAdapter(dict[str, object]) + + +def construct_responses_api_response(payload: Mapping[str, object]) -> ResponsesAPIResponse: + """Build a ``ResponsesAPIResponse``, validating when possible. + + On validation failure, fields required by the model are defaulted and + ``usage`` is coerced, so attribute access on the result behaves as declared. + """ + try: + return ResponsesAPIResponse.model_validate(dict(payload)) + except ValidationError: + verbose_logger.debug( + "Responses API: validation failed for payload %s, falling back to model_construct", + payload, + ) + + output = payload.get("output") + construct: Callable[..., ResponsesAPIResponse] = ResponsesAPIResponse.model_construct + return construct( + **{ + **payload, + "output": output if isinstance(output, list) else [], + "usage": _construct_usage(payload.get("usage")), + } + ) + + +def construct_responses_api_stream_event( + event_model: type[BaseLiteLLMOpenAIResponseObject], + payload: Mapping[str, object], +) -> BaseLiteLLMOpenAIResponseObject: + """Build a Responses API streaming event without validation. + + Terminal events (``response.completed`` / ``.failed`` / ``.incomplete``) + declare a ``ResponsesAPIResponse``; ``model_construct`` alone would leave it + as the raw dict, so construct that nested object explicitly. + """ + construct: Callable[..., BaseLiteLLMOpenAIResponseObject] = event_model.model_construct + response_payload = _as_field_mapping(payload.get("response")) + if response_payload is None: + return construct(**payload) + return construct(**{**payload, "response": construct_responses_api_response(response_payload)}) + + +def _construct_usage(usage: object) -> ResponseAPIUsage | None: + """Coerce a raw ``usage`` payload into ``ResponseAPIUsage``, or drop it.""" + if usage is None or isinstance(usage, ResponseAPIUsage): + return usage + usage_fields = _as_field_mapping(usage) + if usage_fields is None: + return None + + input_tokens = _as_token_count(usage_fields.get("input_tokens")) + output_tokens = _as_token_count(usage_fields.get("output_tokens")) + try: + return ResponseAPIUsage.model_validate( + { + **usage_fields, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": _as_token_count(usage_fields.get("total_tokens")) or input_tokens + output_tokens, + } + ) + except ValidationError: + verbose_logger.debug("Responses API: unparseable usage payload %s, dropping it", usage_fields) + return None + + +def _as_field_mapping(value: object) -> Mapping[str, object] | None: + """Validate an untyped payload into a string-keyed field mapping.""" + try: + return _FIELD_MAPPING_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _as_token_count(value: object) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int): + return value + if isinstance(value, str) and value.isdigit(): + return int(value) + return 0 diff --git a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py index 4581f4af7b6..aa2449f597d 100644 --- a/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py +++ b/tests/test_litellm/llms/volcengine/responses/test_volcengine_responses_transformation.py @@ -295,7 +295,7 @@ class TestVolcengineResponsesAPITransformation: assert event.response.output == [] assert event.response.created_at == 0 - def test_transform_response_api_response_falls_back_to_model_construct(self): + def test_transform_response_api_response_keeps_declared_shape_on_bad_payload(self): config = VolcEngineResponsesAPIConfig() http_response = httpx.Response( status_code=200, @@ -315,7 +315,7 @@ class TestVolcengineResponsesAPITransformation: ) assert result.id == "resp_fallback" - assert result.output == "not-a-list" + assert result.output == [] assert result._hidden_params["headers"].get("x-test") == "1" def test_transform_delete_response_api_request_builds_url(self): diff --git a/tests/test_litellm/responses/test_response_construction.py b/tests/test_litellm/responses/test_response_construction.py new file mode 100644 index 00000000000..6d54f9d1a20 --- /dev/null +++ b/tests/test_litellm/responses/test_response_construction.py @@ -0,0 +1,142 @@ +""" +Regression tests for GitHub issue #34754. + +Providers whose Responses API payloads fail strict validation used to be built +with bare ``model_construct``, leaving ``event.response`` as a raw dict and +``response.usage`` unparsed. Consumers read those as declared, so streaming +requests died with ``AttributeError: 'dict' object has no attribute 'usage'`` +(HTTP 500 mid-stream) or silently dropped the SpendLogs entry. +""" + +import datetime + +import pytest + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.response_construction import construct_responses_api_response +from litellm.router import Router +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIResponse, +) + +# `response.completed` payload that fails validation: `output` is required by +# ResponsesAPIResponse but plenty of providers omit it. +DICT_FORMAT_COMPLETED_CHUNK = { + "type": "response.completed", + "response": { + "id": "resp_34754", + "created_at": 1700000000, + "status": "completed", + "usage": {"input_tokens": 11, "output_tokens": 7, "total_tokens": 18}, + }, +} + + +class _StubStreamingIterator: + def __init__(self, completed_response: object) -> None: + self.completed_response = completed_response + + +def _transform_completed_chunk() -> ResponseCompletedEvent: + event = OpenAIResponsesAPIConfig().transform_streaming_response( + model="gpt-5", + parsed_chunk=DICT_FORMAT_COMPLETED_CHUNK, + logging_obj=None, + ) + assert isinstance(event, ResponseCompletedEvent) + return event + + +def test_unvalidatable_payload_keeps_declared_shape(): + response = construct_responses_api_response(DICT_FORMAT_COMPLETED_CHUNK["response"]) + + assert isinstance(response, ResponsesAPIResponse) + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.input_tokens == 11 + assert response.usage.total_tokens == 18 + assert response.output == [] + assert response.id == "resp_34754" + + +def test_usage_total_tokens_derived_when_missing(): + response = construct_responses_api_response( + {"id": "resp_1", "created_at": 1, "usage": {"input_tokens": 3, "output_tokens": 4}} + ) + + assert isinstance(response.usage, ResponseAPIUsage) + assert response.usage.total_tokens == 7 + + +def test_unparseable_usage_is_dropped_not_left_as_dict(): + response = construct_responses_api_response({"id": "resp_1", "created_at": 1, "usage": "not-a-usage-block"}) + + assert response.usage is None + + +def test_valid_payload_is_validated_not_constructed(): + response = construct_responses_api_response( + { + "id": "resp_1", + "created_at": 1, + "output": [], + "usage": {"input_tokens": 1, "output_tokens": 2, "total_tokens": 3}, + } + ) + + assert isinstance(response.usage, ResponseAPIUsage) + assert response.model_fields_set # validated instance, not a bare model_construct + + +def test_streaming_event_response_is_not_a_dict(): + event = _transform_completed_chunk() + + assert isinstance(event.response, ResponsesAPIResponse) + assert isinstance(event.response.usage, ResponseAPIUsage) + + +def test_router_extracts_partial_usage_from_dict_format_response(): + """Path 1: router fallback usage extraction returned .response.usage -> HTTP 500.""" + event = _transform_completed_chunk() + + usage = Router._extract_partial_responses_usage(_StubStreamingIterator(event)) + + assert isinstance(usage, ResponseAPIUsage) + assert (usage.input_tokens, usage.output_tokens, usage.total_tokens) == (11, 7, 18) + + +def test_streaming_logging_transforms_usage_for_dict_format_response(): + """Path 2: logging silently dropped the SpendLogs entry for these responses.""" + event = _transform_completed_chunk() + logging_obj = LiteLLMLoggingObj( + model="gpt-5", + messages=[], + stream=True, + call_type="aresponses", + start_time=datetime.datetime.now(), + litellm_call_id="34754", + function_id="34754", + ) + + assembled = logging_obj._get_assembled_streaming_response( + result=event, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + is_async=False, + streaming_chunks=[], + ) + + assert assembled is not None + chat_usage = assembled.usage # pyright: ignore[reportAttributeAccessIssue] # set as a dict for serialization + assert (chat_usage["prompt_tokens"], chat_usage["completion_tokens"], chat_usage["total_tokens"]) == (11, 7, 18) + + +@pytest.mark.parametrize("output_value", [None, "not-a-list"]) +def test_output_always_iterable_for_chat_bridge(output_value): + """Path 3: the responses -> chat bridge iterates response.output directly.""" + response = construct_responses_api_response({"id": "resp_1", "created_at": 1, "output": output_value}) + + assert response.output == [] + assert list(response.output) == []