mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(audio): stream the guarded transcript to SDK callers that asked for a stream
This commit is contained in:
parent
020c6bd818
commit
f589986f98
4 changed files with 87 additions and 37 deletions
|
|
@ -21,9 +21,6 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
|
|||
)
|
||||
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
|
||||
|
||||
TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed"
|
||||
|
||||
|
||||
_STREAM_FLAG: Final = TypeAdapter(bool)
|
||||
|
||||
|
||||
|
|
@ -69,25 +66,47 @@ def litellm_exception_mapper(call_details: Mapping[str, object]) -> Transcriptio
|
|||
return map_exception
|
||||
|
||||
|
||||
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
|
||||
def _guarded_transcription_event_payloads(
|
||||
response: TranscriptionResponse,
|
||||
) -> tuple[dict[str, object], dict[str, object]]:
|
||||
text: Final = response.text or ""
|
||||
usage: Final = response.usage
|
||||
languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
|
||||
done_event: Final = {
|
||||
"type": "transcript.text.done",
|
||||
"text": text,
|
||||
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
|
||||
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
|
||||
}
|
||||
return (
|
||||
{"type": "transcript.text.delta", "delta": text},
|
||||
{
|
||||
"type": "transcript.text.done",
|
||||
"text": text,
|
||||
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
|
||||
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
|
||||
return (
|
||||
*(
|
||||
f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
|
||||
for event in ({"type": "transcript.text.delta", "delta": text}, done_event)
|
||||
for event in _guarded_transcription_event_payloads(response)
|
||||
),
|
||||
TRANSCRIPTION_STREAM_DONE_FRAME,
|
||||
)
|
||||
|
||||
|
||||
class GuardedTranscriptionReplay:
|
||||
def __init__(self, response: TranscriptionResponse) -> None:
|
||||
self.response: Final = response
|
||||
self._hidden_params: dict[str, object] = response.hidden_params
|
||||
|
||||
def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
|
||||
return self._events()
|
||||
|
||||
async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
|
||||
delta, done = _guarded_transcription_event_payloads(self.response)
|
||||
yield TranscriptionTextDeltaEvent.model_construct(**delta)
|
||||
yield TranscriptionTextDoneEvent.model_construct(**done)
|
||||
|
||||
|
||||
class TranscriptionStreamLogging(Protocol):
|
||||
def success_handler(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -377,8 +377,8 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
|
|||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
TRANSCRIPTION_RAN_UNSTREAMED,
|
||||
TRANSCRIPTION_STREAM_DONE_FRAME,
|
||||
GuardedTranscriptionReplay,
|
||||
guarded_transcription_events,
|
||||
transcription_stream_requested,
|
||||
)
|
||||
|
|
@ -13894,13 +13894,7 @@ async def audio_transcriptions(
|
|||
)
|
||||
response_headers: Final = {**custom_headers, **(callback_headers or {})}
|
||||
|
||||
ran_unstreamed: Final = (
|
||||
isinstance(response, TranscriptionResponse)
|
||||
and response.hidden_params.get(TRANSCRIPTION_RAN_UNSTREAMED) is True
|
||||
)
|
||||
if transcription_stream_requested(form_data.get("stream")) and (
|
||||
ran_unstreamed or hasattr(response, "__aiter__")
|
||||
):
|
||||
if transcription_stream_requested(form_data.get("stream")) and hasattr(response, "__aiter__"):
|
||||
|
||||
async def transcription_event_stream(
|
||||
stream: AsyncIterator[TranscriptionStreamEvent],
|
||||
|
|
@ -13930,8 +13924,8 @@ async def audio_transcriptions(
|
|||
await close_result
|
||||
|
||||
return StreamingResponse(
|
||||
guarded_transcription_events(response)
|
||||
if isinstance(response, TranscriptionResponse)
|
||||
guarded_transcription_events(response.response)
|
||||
if isinstance(response, GuardedTranscriptionReplay)
|
||||
else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)),
|
||||
media_type="text/event-stream",
|
||||
headers=response_headers,
|
||||
|
|
|
|||
|
|
@ -1143,14 +1143,12 @@ def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> di
|
|||
return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True)
|
||||
|
||||
|
||||
def _marked_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object:
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import TRANSCRIPTION_RAN_UNSTREAMED
|
||||
def _replayed_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object:
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import GuardedTranscriptionReplay
|
||||
|
||||
if not runs_unstreamed or not isinstance(result, TranscriptionResponse):
|
||||
return result
|
||||
marked: Final = result.model_copy()
|
||||
marked.hidden_params = {**result.hidden_params, TRANSCRIPTION_RAN_UNSTREAMED: True}
|
||||
return marked
|
||||
return GuardedTranscriptionReplay(result)
|
||||
|
||||
|
||||
def _is_converted_stream_result(result: object) -> bool:
|
||||
|
|
@ -2448,7 +2446,7 @@ def client(original_function):
|
|||
if _is_converted_stream_result(_caching_handler_response.cached_result):
|
||||
logging_obj.stream = True
|
||||
logging_obj.model_call_details["stream"] = True
|
||||
return _marked_when_run_unstreamed(
|
||||
return _replayed_when_run_unstreamed(
|
||||
_caching_handler_response.cached_result, transcription_runs_unstreamed
|
||||
)
|
||||
|
||||
|
|
@ -2561,15 +2559,12 @@ def client(original_function):
|
|||
# Only run if call_type is a valid value in CallTypes
|
||||
_call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type)
|
||||
if _call_type_enum is not None:
|
||||
result = _marked_when_run_unstreamed(
|
||||
await async_post_call_success_deployment_hook(
|
||||
request_data=_guardrail_view_of_requested_stream(call_kwargs)
|
||||
if transcription_runs_unstreamed
|
||||
else call_kwargs,
|
||||
response=result,
|
||||
call_type=_call_type_enum,
|
||||
),
|
||||
transcription_runs_unstreamed,
|
||||
result = await async_post_call_success_deployment_hook(
|
||||
request_data=_guardrail_view_of_requested_stream(call_kwargs)
|
||||
if transcription_runs_unstreamed
|
||||
else call_kwargs,
|
||||
response=result,
|
||||
call_type=_call_type_enum,
|
||||
)
|
||||
|
||||
## Add response to cache
|
||||
|
|
@ -2618,7 +2613,7 @@ def client(original_function):
|
|||
is_litellm_internal_call=_is_litellm_internal_call,
|
||||
)
|
||||
|
||||
return result
|
||||
return _replayed_when_run_unstreamed(result, transcription_runs_unstreamed)
|
||||
except Exception as e:
|
||||
traceback_exception: Final = traceback.format_exc()
|
||||
# Reuse the timestamp taken right when the deployment call itself failed, before
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import pytest
|
|||
from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
litellm_exception_mapper,
|
||||
|
|
@ -24,6 +25,7 @@ from litellm.llms.openai.transcriptions.gpt_transformation import (
|
|||
OpenAIGPTTranscribeAudioTranscriptionConfig,
|
||||
)
|
||||
from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
from litellm.utils import get_optional_params_transcription
|
||||
|
||||
|
|
@ -682,6 +684,46 @@ async def test_atranscription_stream_bills_the_provider_duration_like_the_unstre
|
|||
assert streamed_spend == unstreamed_spend
|
||||
|
||||
|
||||
class _TranscriptRedactor(CustomGuardrail):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call)
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self, data: dict[str, object], user_api_key_dict: object, response: object
|
||||
) -> None:
|
||||
if isinstance(response, TranscriptionResponse):
|
||||
response.text = "[REDACTED]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_atranscription_stream_under_a_post_call_guardrail_streams_the_guarded_transcript(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread()))
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()])
|
||||
client: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)),
|
||||
)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe",
|
||||
file=_one_second_wav(),
|
||||
stream=True,
|
||||
client=client,
|
||||
guardrails=["transcript-redactor"],
|
||||
)
|
||||
events: Final = [(event.type, event.model_dump(exclude={"type"}, exclude_unset=True)) async for event in stream]
|
||||
await client.close()
|
||||
|
||||
assert events == [
|
||||
("transcript.text.delta", {"delta": "[REDACTED]"}),
|
||||
("transcript.text.done", {"text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}),
|
||||
]
|
||||
|
||||
|
||||
def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue