diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 44b1667dd20..4650b7db2f2 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -21,9 +21,6 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ) from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage -TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed" - - _STREAM_FLAG: Final = TypeAdapter(bool) @@ -69,25 +66,47 @@ def litellm_exception_mapper(call_details: Mapping[str, object]) -> Transcriptio return map_exception -def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]: +def _guarded_transcription_event_payloads( + response: TranscriptionResponse, +) -> tuple[dict[str, object], dict[str, object]]: text: Final = response.text or "" usage: Final = response.usage languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None)) - done_event: Final = { - "type": "transcript.text.done", - "text": text, - **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}), - **({} if languages is None else {"languages": [language.model_dump() for language in languages]}), - } + return ( + {"type": "transcript.text.delta", "delta": text}, + { + "type": "transcript.text.done", + "text": text, + **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}), + **({} if languages is None else {"languages": [language.model_dump() for language in languages]}), + }, + ) + + +def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]: return ( *( f"data: {json.dumps(event, separators=(',', ':'))}\n\n" - for event in ({"type": "transcript.text.delta", "delta": text}, done_event) + for event in _guarded_transcription_event_payloads(response) ), TRANSCRIPTION_STREAM_DONE_FRAME, ) +class GuardedTranscriptionReplay: + def __init__(self, response: TranscriptionResponse) -> None: + self.response: Final = response + self._hidden_params: dict[str, object] = response.hidden_params + + def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]: + return self._events() + + async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]: + delta, done = _guarded_transcription_event_payloads(self.response) + yield TranscriptionTextDeltaEvent.model_construct(**delta) + yield TranscriptionTextDoneEvent.model_construct(**done) + + class TranscriptionStreamLogging(Protocol): def success_handler( self, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c3b2ae4bad3..0942a7a4ad7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -377,8 +377,8 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( - TRANSCRIPTION_RAN_UNSTREAMED, TRANSCRIPTION_STREAM_DONE_FRAME, + GuardedTranscriptionReplay, guarded_transcription_events, transcription_stream_requested, ) @@ -13894,13 +13894,7 @@ async def audio_transcriptions( ) response_headers: Final = {**custom_headers, **(callback_headers or {})} - ran_unstreamed: Final = ( - isinstance(response, TranscriptionResponse) - and response.hidden_params.get(TRANSCRIPTION_RAN_UNSTREAMED) is True - ) - if transcription_stream_requested(form_data.get("stream")) and ( - ran_unstreamed or hasattr(response, "__aiter__") - ): + if transcription_stream_requested(form_data.get("stream")) and hasattr(response, "__aiter__"): async def transcription_event_stream( stream: AsyncIterator[TranscriptionStreamEvent], @@ -13930,8 +13924,8 @@ async def audio_transcriptions( await close_result return StreamingResponse( - guarded_transcription_events(response) - if isinstance(response, TranscriptionResponse) + guarded_transcription_events(response.response) + if isinstance(response, GuardedTranscriptionReplay) else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)), media_type="text/event-stream", headers=response_headers, diff --git a/litellm/utils.py b/litellm/utils.py index 21ef4d793bb..64b7eedcf8c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1143,14 +1143,12 @@ def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> di return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True) -def _marked_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object: - from litellm.litellm_core_utils.audio_utils.transcription_streaming import TRANSCRIPTION_RAN_UNSTREAMED +def _replayed_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import GuardedTranscriptionReplay if not runs_unstreamed or not isinstance(result, TranscriptionResponse): return result - marked: Final = result.model_copy() - marked.hidden_params = {**result.hidden_params, TRANSCRIPTION_RAN_UNSTREAMED: True} - return marked + return GuardedTranscriptionReplay(result) def _is_converted_stream_result(result: object) -> bool: @@ -2448,7 +2446,7 @@ def client(original_function): if _is_converted_stream_result(_caching_handler_response.cached_result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True - return _marked_when_run_unstreamed( + return _replayed_when_run_unstreamed( _caching_handler_response.cached_result, transcription_runs_unstreamed ) @@ -2561,15 +2559,12 @@ def client(original_function): # Only run if call_type is a valid value in CallTypes _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: - result = _marked_when_run_unstreamed( - await async_post_call_success_deployment_hook( - request_data=_guardrail_view_of_requested_stream(call_kwargs) - if transcription_runs_unstreamed - else call_kwargs, - response=result, - call_type=_call_type_enum, - ), - transcription_runs_unstreamed, + result = await async_post_call_success_deployment_hook( + request_data=_guardrail_view_of_requested_stream(call_kwargs) + if transcription_runs_unstreamed + else call_kwargs, + response=result, + call_type=_call_type_enum, ) ## Add response to cache @@ -2618,7 +2613,7 @@ def client(original_function): is_litellm_internal_call=_is_litellm_internal_call, ) - return result + return _replayed_when_run_unstreamed(result, transcription_runs_unstreamed) except Exception as e: traceback_exception: Final = traceback.format_exc() # Reuse the timestamp taken right when the deployment call itself failed, before diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index fa03cfe8f46..eff4be19415 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -13,6 +13,7 @@ import pytest from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( litellm_exception_mapper, @@ -24,6 +25,7 @@ from litellm.llms.openai.transcriptions.gpt_transformation import ( OpenAIGPTTranscribeAudioTranscriptionConfig, ) from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import TranscriptionResponse from litellm.utils import get_optional_params_transcription @@ -682,6 +684,46 @@ async def test_atranscription_stream_bills_the_provider_duration_like_the_unstre assert streamed_spend == unstreamed_spend +class _TranscriptRedactor(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call) + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + if isinstance(response, TranscriptionResponse): + response.text = "[REDACTED]" + + +@pytest.mark.asyncio +async def test_atranscription_stream_under_a_post_call_guardrail_streams_the_guarded_transcript( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", + file=_one_second_wav(), + stream=True, + client=client, + guardrails=["transcript-redactor"], + ) + events: Final = [(event.type, event.model_dump(exclude={"type"}, exclude_unset=True)) async for event in stream] + await client.close() + + assert events == [ + ("transcript.text.delta", {"delta": "[REDACTED]"}), + ("transcript.text.done", {"text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}), + ] + + def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call( local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: