diff --git a/litellm/utils.py b/litellm/utils.py index 734522c0c6a..22b81c25daf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -853,6 +853,40 @@ def _is_converted_stream_result(result: object) -> bool: return isinstance(result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator)) +async def _run_success_deployment_hook_on_converted_chat_stream( + result: object, request_data: dict[str, object], call_type: str +) -> object: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.llms.base_llm.base_model_iterator import MockResponseIterator + + if not isinstance(result, CustomStreamWrapper): + return result + completion_stream: Final = result.completion_stream + if not isinstance(completion_stream, MockResponseIterator): + return result + call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) + if call_type_enum is None: + return result + hooked: Final = await async_post_call_success_deployment_hook( + request_data=request_data, + response=completion_stream.model_response, + call_type=call_type_enum, + ) + if not isinstance(hooked, ModelResponse) or hooked is completion_stream.model_response: + return result + rewrapped: Final = CustomStreamWrapper( + completion_stream=MockResponseIterator(model_response=hooked, json_mode=completion_stream.json_mode), + model=result.model, + custom_llm_provider=result.custom_llm_provider, + logging_obj=result.logging_obj, + stream_options=result.stream_options, + make_call=result.make_call, + count_prompt_tokens=result.count_prompt_tokens, + ) + rewrapped.set_logging_event_loop(loop=result.logging_loop) + return rewrapped + + # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. def function_setup( original_function: str, @@ -1956,17 +1990,25 @@ def client(original_function): raise end_time = datetime.datetime.now() - if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): + streaming_requested: Final = _is_streaming_request(kwargs=kwargs, call_type=call_type) + if streaming_requested or _is_converted_stream_result(result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True + stream_result: Final = ( + result + if streaming_requested + else await _run_success_deployment_hook_on_converted_chat_stream( + result=result, request_data=kwargs, call_type=call_type + ) + ) if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks: Final = [] - for idx, chunk in enumerate(result): + for idx, chunk in enumerate(stream_result): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: _update_response_metadata( - result=result, + result=stream_result, logging_obj=logging_obj, model=model, kwargs=kwargs, @@ -1974,7 +2016,7 @@ def client(original_function): end_time=end_time, ) return _llm_caching_handler.wrap_streaming_result_for_cache( - result=result, + result=stream_result, call_type=call_type, ) elif call_type == CallTypes.arealtime.value: diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 46149589371..5da588ceb2e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -32,6 +32,7 @@ from litellm._logging import ( from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.thread_pool_executor import executor as logging_executor +from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.proxy.utils import is_valid_api_key from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY @@ -39,6 +40,7 @@ from litellm.types.utils import ( CallTypes, Delta, LlmProviders, + ModelResponse, ModelResponseStream, PromptTokensDetailsWrapper, StreamingChoices, @@ -4437,6 +4439,76 @@ async def test_wrapper_async_logs_converted_chat_stream_with_standard_logging_ob assert success_kwargs["stream"] is True +class _RewritingSuccessDeploymentHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_responses: list[object] = [] + + async def async_post_call_success_deployment_hook( + self, request_data: dict[str, object], response: object, call_type: CallTypes | None + ) -> ModelResponse | None: + self.seen_responses.append(response) + if not isinstance(response, ModelResponse): + return None + rewritten: Final = response.model_copy(deep=True) + rewritten.choices[0].message.content = "rewritten by deployment hook" + return rewritten + + +@pytest.mark.asyncio +async def test_wrapper_async_runs_success_deployment_hook_on_converted_chat_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _install_converted_stream_callbacks(monkeypatch) + hook: Final = _RewritingSuccessDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [_ConvertStreamDeploymentHook(), hook]) + + response: Final = await litellm.acompletion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + stream=True, + mock_response="converted stream body", + num_retries=0, + ) + assert isinstance(response, CustomStreamWrapper) + chunks: Final = [chunk async for chunk in response] + + assert len(hook.seen_responses) == 1 + seen: Final = hook.seen_responses[0] + assert isinstance(seen, ModelResponse) + assert seen.choices[0].message.content == "converted stream body" + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "rewritten by deployment hook" + + +@pytest.mark.asyncio +@respx.mock +async def test_wrapper_async_leaves_success_deployment_hook_off_requested_fake_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + hook: Final = _RewritingSuccessDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [hook]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx.post("http://fake-stream.invalid/api/v1/run/flow-1").respond( + json={"outputs": [{"outputs": [{"results": {"message": {"text": "plain stream body"}}}]}]} + ) + + response: Final = await litellm.acompletion( + model="langflow/flow-1", + api_base="http://fake-stream.invalid", + api_key="fake-key", + messages=[{"role": "user", "content": "hi"}], + stream=True, + num_retries=0, + ) + assert isinstance(response, CustomStreamWrapper) + assert isinstance(response.completion_stream, MockResponseIterator) + chunks: Final = [chunk async for chunk in response] + + assert hook.seen_responses == [] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "plain stream body" + + @pytest.mark.asyncio @respx.mock async def test_wrapper_async_logs_converted_responses_stream_with_standard_logging_object(