diff --git a/litellm/utils.py b/litellm/utils.py index 18df5e2abf7..9031e23b39e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -846,6 +846,15 @@ def _is_streaming_response_for_correlation(result: object) -> bool: return isinstance(result, CustomStreamWrapper) +def _is_converted_stream_result(result: object) -> bool: + """True if `result` is a lazy stream wrapper the caller must iterate, even when a deployment + hook downgraded `kwargs["stream"]` to False for the provider call.""" + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + return isinstance(result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator)) + + # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. def function_setup( original_function: str, @@ -1946,10 +1955,9 @@ def client(original_function): raise end_time = datetime.datetime.now() - if _is_streaming_request( - kwargs=kwargs, - call_type=call_type, - ): + if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): + logging_obj.stream = True + logging_obj.model_call_details["stream"] = True if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks: Final = [] for idx, chunk in enumerate(result): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ae0e08ebfb1..7c33cf40bc8 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -32,6 +32,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.thread_pool_executor import executor as logging_executor from litellm.proxy.utils import is_valid_api_key +from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY from litellm.types.utils import ( CallTypes, Delta, @@ -44,6 +45,7 @@ from litellm.types.utils import ( from litellm.types.utils import all_litellm_params, bedrock_batch_litellm_params from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.utils import ( + CustomStreamWrapper, ProviderConfigManager, TextCompletionStreamWrapper, _check_provider_match, @@ -5307,6 +5309,121 @@ async def test_wrapper_async_restores_originating_task_context_after_success(mon session_id_var.set("") +class _ConvertStreamDeploymentHook(CustomLogger): + """Headroom-style interception: downgrade stream=True to a non-streaming provider call.""" + + async def async_pre_call_deployment_hook( + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict[str, object] | None: + if not kwargs.get("stream"): + return None + return {**kwargs, "stream": False, HEADROOM_CONVERTED_STREAM_KEY: True} + + +class _SuccessKwargsCapture(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.success_kwargs: list[dict[str, object]] = [] + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.success_kwargs.append(kwargs) + + +def _install_converted_stream_callbacks(monkeypatch: pytest.MonkeyPatch) -> _SuccessKwargsCapture: + capture: Final = _SuccessKwargsCapture() + monkeypatch.setattr(litellm, "callbacks", [_ConvertStreamDeploymentHook(), capture]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + return capture + + +async def _wait_for_success_kwargs(capture: _SuccessKwargsCapture) -> dict[str, object]: + for _ in range(50): + if capture.success_kwargs: + break + await asyncio.sleep(0.05) + (success_kwargs,) = capture.success_kwargs + return success_kwargs + + +@pytest.mark.asyncio +async def test_wrapper_async_logs_converted_chat_stream_with_standard_logging_object( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression LIT-7729: the fake CustomStreamWrapper hit the non-streaming success path, which + built no standard_logging_object and deduped the wrapper's own end-of-stream dispatch.""" + capture: Final = _install_converted_stream_callbacks(monkeypatch) + + response: Final = await litellm.acompletion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + stream=True, + mock_response="converted stream body", + num_retries=0, + ) + assert isinstance(response, CustomStreamWrapper) + chunks: Final = [chunk async for chunk in response] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "converted stream body" + + success_kwargs: Final = await _wait_for_success_kwargs(capture) + standard_logging_object: Final = success_kwargs["standard_logging_object"] + assert isinstance(standard_logging_object, dict) + assert standard_logging_object["response_cost"] > 0 + assert standard_logging_object["stream"] is True + assert success_kwargs["stream"] is True + + +@pytest.mark.asyncio +@respx.mock +async def test_wrapper_async_logs_converted_responses_stream_with_standard_logging_object( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression LIT-7729, Responses surface: the fake MockResponsesAPIStreamingIterator took the + same non-streaming success path and lost its standard_logging_object.""" + from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator + + capture: Final = _install_converted_stream_callbacks(monkeypatch) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx.post("https://api.openai.com/v1/responses").respond( + json={ + "id": "resp_converted", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_converted", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "converted stream body", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, + } + ) + + response: Final = await litellm.aresponses( + model="openai/gpt-5.6", input="hi", stream=True, api_key="sk-test", num_retries=0 + ) + assert isinstance(response, BaseResponsesAPIStreamingIterator) + events: Final = [event async for event in response] + assert events[-1].type == "response.completed" + + success_kwargs: Final = await _wait_for_success_kwargs(capture) + standard_logging_object: Final = success_kwargs["standard_logging_object"] + assert isinstance(standard_logging_object, dict) + assert standard_logging_object["response_cost"] > 0 + assert standard_logging_object["stream"] is True + assert success_kwargs["stream"] is True + + def test_function_setup_failure_after_logging_construction_restores_context(monkeypatch): """If function_setup() constructs Logging() (which already mutated trace_id_var/session_id_var in __init__) but then raises before returning,