fix(audio): stream the guarded transcript to SDK callers that asked for a stream

This commit is contained in:
mateo-berri 2026-11-02 17:25:12 -08:00
parent 020c6bd818
commit f589986f98
4 changed files with 87 additions and 37 deletions

View file

@ -21,9 +21,6 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
)
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed"
_STREAM_FLAG: Final = TypeAdapter(bool)
@ -69,25 +66,47 @@ def litellm_exception_mapper(call_details: Mapping[str, object]) -> Transcriptio
return map_exception
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
def _guarded_transcription_event_payloads(
response: TranscriptionResponse,
) -> tuple[dict[str, object], dict[str, object]]:
text: Final = response.text or ""
usage: Final = response.usage
languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
done_event: Final = {
"type": "transcript.text.done",
"text": text,
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
}
return (
{"type": "transcript.text.delta", "delta": text},
{
"type": "transcript.text.done",
"text": text,
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
},
)
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
return (
*(
f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
for event in ({"type": "transcript.text.delta", "delta": text}, done_event)
for event in _guarded_transcription_event_payloads(response)
),
TRANSCRIPTION_STREAM_DONE_FRAME,
)
class GuardedTranscriptionReplay:
def __init__(self, response: TranscriptionResponse) -> None:
self.response: Final = response
self._hidden_params: dict[str, object] = response.hidden_params
def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
return self._events()
async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
delta, done = _guarded_transcription_event_payloads(self.response)
yield TranscriptionTextDeltaEvent.model_construct(**delta)
yield TranscriptionTextDoneEvent.model_construct(**done)
class TranscriptionStreamLogging(Protocol):
def success_handler(
self,

View file

@ -377,8 +377,8 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
TRANSCRIPTION_RAN_UNSTREAMED,
TRANSCRIPTION_STREAM_DONE_FRAME,
GuardedTranscriptionReplay,
guarded_transcription_events,
transcription_stream_requested,
)
@ -13894,13 +13894,7 @@ async def audio_transcriptions(
)
response_headers: Final = {**custom_headers, **(callback_headers or {})}
ran_unstreamed: Final = (
isinstance(response, TranscriptionResponse)
and response.hidden_params.get(TRANSCRIPTION_RAN_UNSTREAMED) is True
)
if transcription_stream_requested(form_data.get("stream")) and (
ran_unstreamed or hasattr(response, "__aiter__")
):
if transcription_stream_requested(form_data.get("stream")) and hasattr(response, "__aiter__"):
async def transcription_event_stream(
stream: AsyncIterator[TranscriptionStreamEvent],
@ -13930,8 +13924,8 @@ async def audio_transcriptions(
await close_result
return StreamingResponse(
guarded_transcription_events(response)
if isinstance(response, TranscriptionResponse)
guarded_transcription_events(response.response)
if isinstance(response, GuardedTranscriptionReplay)
else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)),
media_type="text/event-stream",
headers=response_headers,

View file

@ -1143,14 +1143,12 @@ def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> di
return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True)
def _marked_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object:
from litellm.litellm_core_utils.audio_utils.transcription_streaming import TRANSCRIPTION_RAN_UNSTREAMED
def _replayed_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object:
from litellm.litellm_core_utils.audio_utils.transcription_streaming import GuardedTranscriptionReplay
if not runs_unstreamed or not isinstance(result, TranscriptionResponse):
return result
marked: Final = result.model_copy()
marked.hidden_params = {**result.hidden_params, TRANSCRIPTION_RAN_UNSTREAMED: True}
return marked
return GuardedTranscriptionReplay(result)
def _is_converted_stream_result(result: object) -> bool:
@ -2448,7 +2446,7 @@ def client(original_function):
if _is_converted_stream_result(_caching_handler_response.cached_result):
logging_obj.stream = True
logging_obj.model_call_details["stream"] = True
return _marked_when_run_unstreamed(
return _replayed_when_run_unstreamed(
_caching_handler_response.cached_result, transcription_runs_unstreamed
)
@ -2561,15 +2559,12 @@ def client(original_function):
# Only run if call_type is a valid value in CallTypes
_call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type)
if _call_type_enum is not None:
result = _marked_when_run_unstreamed(
await async_post_call_success_deployment_hook(
request_data=_guardrail_view_of_requested_stream(call_kwargs)
if transcription_runs_unstreamed
else call_kwargs,
response=result,
call_type=_call_type_enum,
),
transcription_runs_unstreamed,
result = await async_post_call_success_deployment_hook(
request_data=_guardrail_view_of_requested_stream(call_kwargs)
if transcription_runs_unstreamed
else call_kwargs,
response=result,
call_type=_call_type_enum,
)
## Add response to cache
@ -2618,7 +2613,7 @@ def client(original_function):
is_litellm_internal_call=_is_litellm_internal_call,
)
return result
return _replayed_when_run_unstreamed(result, transcription_runs_unstreamed)
except Exception as e:
traceback_exception: Final = traceback.format_exc()
# Reuse the timestamp taken right when the deployment call itself failed, before

View file

@ -13,6 +13,7 @@ import pytest
from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI
import litellm
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
litellm_exception_mapper,
@ -24,6 +25,7 @@ from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTTranscribeAudioTranscriptionConfig,
)
from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import TranscriptionResponse
from litellm.utils import get_optional_params_transcription
@ -682,6 +684,46 @@ async def test_atranscription_stream_bills_the_provider_duration_like_the_unstre
assert streamed_spend == unstreamed_spend
class _TranscriptRedactor(CustomGuardrail):
def __init__(self) -> None:
super().__init__(guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call)
async def async_post_call_success_hook(
self, data: dict[str, object], user_api_key_dict: object, response: object
) -> None:
if isinstance(response, TranscriptionResponse):
response.text = "[REDACTED]"
@pytest.mark.asyncio
async def test_atranscription_stream_under_a_post_call_guardrail_streams_the_guarded_transcript(
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
async def send_response(request: httpx.Request) -> httpx.Response:
return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread()))
monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()])
client: Final = AsyncOpenAI(
api_key="sk-test",
base_url="https://example.com/v1",
http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)),
)
stream: Final = await litellm.atranscription(
model="openai/gpt-transcribe",
file=_one_second_wav(),
stream=True,
client=client,
guardrails=["transcript-redactor"],
)
events: Final = [(event.type, event.model_dump(exclude={"type"}, exclude_unset=True)) async for event in stream]
await client.close()
assert events == [
("transcript.text.delta", {"delta": "[REDACTED]"}),
("transcript.text.done", {"text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}),
]
def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call(
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None: