mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge ee9d0cf6a9 into 901b7f762f
This commit is contained in:
commit
c7fdde7ea7
88 changed files with 12244 additions and 713 deletions
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -441,6 +441,7 @@ jobs:
|
|||
tests/unit/ocr
|
||||
tests/unit/passthrough
|
||||
tests/unit/rag
|
||||
tests/unit/cookbook
|
||||
tests/unit/realtime_api
|
||||
tests/unit/rerank_api
|
||||
tests/unit/search
|
||||
|
|
|
|||
245
cookbook/gpt_realtime_translate.py
Normal file
245
cookbook/gpt_realtime_translate.py
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
from collections.abc import Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlencode, urlsplit, urlunsplit
|
||||
|
||||
import websockets
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
SAMPLE_RATE = 24_000
|
||||
CHANNELS = 1
|
||||
SAMPLE_WIDTH = 2
|
||||
CHUNK_DURATION_SECONDS = 0.1
|
||||
CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
|
||||
RESPONSE_TIMEOUT_SECONDS = 30.0
|
||||
AUDIO_EVENT_TYPES = frozenset(
|
||||
{
|
||||
"session.output_audio.delta",
|
||||
"response.audio.delta",
|
||||
"response.output_audio.delta",
|
||||
}
|
||||
)
|
||||
TRANSCRIPT_EVENT_TYPES = frozenset(
|
||||
{
|
||||
"session.output_transcript.delta",
|
||||
"response.text.delta",
|
||||
"response.output_audio_transcript.delta",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Settings:
|
||||
input_wav: Path
|
||||
output_wav: Path
|
||||
base_url: str
|
||||
model: str
|
||||
target_language: str
|
||||
trailing_silence_seconds: float
|
||||
api_key: str
|
||||
|
||||
|
||||
def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
|
||||
sys.stdout.write(f"{message}{end}")
|
||||
if flush:
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def write_stderr(message: str) -> None:
|
||||
sys.stderr.write(f"{message}\n")
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
|
||||
)
|
||||
parser.add_argument("input_wav", type=Path)
|
||||
parser.add_argument("--output", type=Path, default=Path("translated.wav"))
|
||||
parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
|
||||
parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
|
||||
parser.add_argument("--target-language", default="fr")
|
||||
parser.add_argument("--trailing-silence", type=float, default=1.5)
|
||||
parsed = parser.parse_args(argv)
|
||||
api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
|
||||
if not api_key:
|
||||
return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
|
||||
if parsed.trailing_silence < 0:
|
||||
return "--trailing-silence must be zero or greater"
|
||||
return Settings(
|
||||
input_wav=parsed.input_wav,
|
||||
output_wav=parsed.output,
|
||||
base_url=parsed.base_url,
|
||||
model=parsed.model,
|
||||
target_language=parsed.target_language,
|
||||
trailing_silence_seconds=parsed.trailing_silence,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
|
||||
def translation_url(base_url: str, model: str) -> str | None:
|
||||
parsed = urlsplit(base_url.rstrip("/"))
|
||||
scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
|
||||
if not scheme or not parsed.netloc:
|
||||
return None
|
||||
base_path = parsed.path.rstrip("/")
|
||||
realtime_path = (
|
||||
f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
|
||||
)
|
||||
return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
|
||||
|
||||
|
||||
def read_pcm16_wav(path: Path) -> bytes | str:
|
||||
try:
|
||||
with wave.open(str(path), "rb") as source:
|
||||
actual_format = (
|
||||
source.getnchannels(),
|
||||
source.getsampwidth(),
|
||||
source.getframerate(),
|
||||
source.getcomptype(),
|
||||
)
|
||||
expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
|
||||
if actual_format != expected_format:
|
||||
return (
|
||||
f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
|
||||
f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
|
||||
f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
|
||||
)
|
||||
return source.readframes(source.getnframes())
|
||||
except (OSError, EOFError, wave.Error) as exc:
|
||||
return f"Unable to read {path}: {exc}"
|
||||
|
||||
|
||||
def audio_chunks(audio: bytes) -> Iterator[bytes]:
|
||||
return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
|
||||
|
||||
|
||||
def audio_message(audio: bytes) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "session.input_audio_buffer.append",
|
||||
"audio": base64.b64encode(audio).decode("ascii"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
|
||||
await connection.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {"audio": {"output": {"language": target_language}}},
|
||||
}
|
||||
)
|
||||
)
|
||||
while True:
|
||||
raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
|
||||
event = json.loads(raw_event)
|
||||
event_type = event.get("type")
|
||||
if event_type == "session.created":
|
||||
write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
|
||||
if event_type == "session.updated":
|
||||
return None
|
||||
if event_type == "error":
|
||||
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
|
||||
|
||||
|
||||
async def send_audio(
|
||||
connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
|
||||
) -> None:
|
||||
silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
|
||||
try:
|
||||
for chunk in audio_chunks(pcm + silence):
|
||||
await connection.send(audio_message(chunk))
|
||||
await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
|
||||
await connection.send(json.dumps({"type": "session.close"}))
|
||||
finally:
|
||||
finished.set()
|
||||
|
||||
|
||||
async def receive_translation(
|
||||
connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
|
||||
) -> str | None:
|
||||
audio_received = asyncio.Event()
|
||||
try:
|
||||
with wave.open(str(output_path), "wb") as output:
|
||||
output.setnchannels(CHANNELS)
|
||||
output.setsampwidth(SAMPLE_WIDTH)
|
||||
output.setframerate(SAMPLE_RATE)
|
||||
write_stdout("Translation: ", end="", flush=True)
|
||||
while True:
|
||||
try:
|
||||
raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
|
||||
except TimeoutError:
|
||||
if not sender_finished.is_set():
|
||||
continue
|
||||
return (
|
||||
f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
|
||||
)
|
||||
event = json.loads(raw_event)
|
||||
event_type = event.get("type")
|
||||
if event_type in AUDIO_EVENT_TYPES:
|
||||
output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
|
||||
audio_received.set()
|
||||
elif event_type in TRANSCRIPT_EVENT_TYPES:
|
||||
write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
|
||||
elif event_type == "error":
|
||||
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
|
||||
elif event_type == "session.closed":
|
||||
write_stdout()
|
||||
return None if audio_received.is_set() else "The translation stream ended without translated audio"
|
||||
except (OSError, wave.Error) as exc:
|
||||
return f"Unable to write {output_path}: {exc}"
|
||||
|
||||
|
||||
async def translate(settings: Settings, pcm: bytes) -> str | None:
|
||||
url = translation_url(settings.base_url, settings.model)
|
||||
if not url:
|
||||
return f"Invalid --base-url: {settings.base_url}"
|
||||
sender_finished = asyncio.Event()
|
||||
try:
|
||||
async with websockets.connect(
|
||||
url,
|
||||
additional_headers={"Authorization": f"Bearer {settings.api_key}"},
|
||||
proxy=None,
|
||||
open_timeout=20,
|
||||
close_timeout=5,
|
||||
) as connection:
|
||||
configuration_error = await configure_session(connection, settings.target_language)
|
||||
if configuration_error:
|
||||
return configuration_error
|
||||
async with asyncio.TaskGroup() as tasks:
|
||||
receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
|
||||
tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
|
||||
return receiver.result()
|
||||
except Exception as exc:
|
||||
return f"Translation failed: {type(exc).__name__}: {exc}"
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
settings = parse_args(argv)
|
||||
if isinstance(settings, str):
|
||||
write_stderr(settings)
|
||||
return 2
|
||||
pcm = read_pcm16_wav(settings.input_wav)
|
||||
if isinstance(pcm, str):
|
||||
write_stderr(pcm)
|
||||
return 2
|
||||
error = asyncio.run(translate(settings, pcm))
|
||||
if error:
|
||||
write_stderr(error)
|
||||
return 1
|
||||
write_stdout(f"Translated audio: {settings.output_wav}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -93,6 +93,12 @@ pub enum Endpoint {
|
|||
Realtime,
|
||||
#[serde(rename = "/v1/realtime/transcription_sessions")]
|
||||
RealtimeTranscriptionSessions,
|
||||
#[serde(rename = "/v1/realtime/translations")]
|
||||
RealtimeTranslations,
|
||||
#[serde(rename = "/v1/realtime/translations/calls")]
|
||||
RealtimeTranslationCalls,
|
||||
#[serde(rename = "/v1/realtime/translations/client_secrets")]
|
||||
RealtimeTranslationClientSecrets,
|
||||
#[serde(rename = "/v1/videos")]
|
||||
Videos,
|
||||
#[serde(rename = "/v1/ocr")]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ _apply_process_defaults()
|
|||
import threading
|
||||
import os
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Sequence
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
|
|
@ -1562,7 +1562,9 @@ from .realtime_api.main import (
|
|||
_arealtime,
|
||||
acreate_realtime_client_secret,
|
||||
acreate_realtime_transcription_session,
|
||||
acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
|
||||
arealtime_calls,
|
||||
arealtime_translation_calls as arealtime_translation_calls,
|
||||
)
|
||||
from .responses.main import _aresponses_websocket
|
||||
|
||||
|
|
@ -1808,6 +1810,9 @@ if TYPE_CHECKING:
|
|||
from .llms.vertex_ai.rerank.transformation import (
|
||||
VertexAIRerankConfig as VertexAIRerankConfig,
|
||||
)
|
||||
from .llms.together_ai.chat.transformation import (
|
||||
TogetherAIChatConfig as TogetherAIChatConfig,
|
||||
)
|
||||
from .llms.fireworks_ai.rerank.transformation import (
|
||||
FireworksAIRerankConfig as FireworksAIRerankConfig,
|
||||
)
|
||||
|
|
@ -1853,9 +1858,6 @@ if TYPE_CHECKING:
|
|||
BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
|
||||
)
|
||||
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
|
||||
from .llms.together_ai.chat.transformation import (
|
||||
TogetherAIChatConfig as TogetherAIChatConfig,
|
||||
)
|
||||
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
|
||||
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig as VertexGeminiConfig,
|
||||
|
|
@ -2014,6 +2016,9 @@ if TYPE_CHECKING:
|
|||
from .llms.xai.responses.transformation import (
|
||||
XAIResponsesAPIConfig as XAIResponsesAPIConfig,
|
||||
)
|
||||
from .llms.vertex_ai.interactions.transformation import (
|
||||
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
|
||||
)
|
||||
from .llms.litellm_proxy.responses.transformation import (
|
||||
LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
|
||||
)
|
||||
|
|
@ -2041,9 +2046,6 @@ if TYPE_CHECKING:
|
|||
from .llms.gemini.interactions.transformation import (
|
||||
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
|
||||
)
|
||||
from .llms.vertex_ai.interactions.transformation import (
|
||||
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
|
||||
)
|
||||
from .llms.openai.chat.o_series_transformation import (
|
||||
OpenAIOSeriesConfig as OpenAIOSeriesConfig,
|
||||
OpenAIOSeriesConfig as OpenAIO1Config,
|
||||
|
|
|
|||
|
|
@ -968,7 +968,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
|
|||
|
||||
OPENAI_TRANSCRIPTION_PARAMS: Final = [
|
||||
"language",
|
||||
"languages",
|
||||
"keywords",
|
||||
"response_format",
|
||||
"stream",
|
||||
"timestamp_granularities",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ from litellm.constants import (
|
|||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
|
||||
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
|
||||
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
|
||||
|
|
@ -319,12 +320,41 @@ def _get_additional_costs(
|
|||
return None
|
||||
|
||||
|
||||
def _transcription_usage_has_token_details(
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second",
|
||||
"output_cost_per_second",
|
||||
"input_cost_per_audio_token",
|
||||
"output_cost_per_audio_token",
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
]
|
||||
_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"input_cost_per_audio_token",
|
||||
"output_cost_per_audio_token",
|
||||
)
|
||||
_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
|
||||
"input_cost_per_second",
|
||||
"output_cost_per_second",
|
||||
)
|
||||
|
||||
|
||||
def _transcription_uses_token_pricing(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
usage_block: Usage | None,
|
||||
) -> bool:
|
||||
if usage_block is None:
|
||||
return False
|
||||
|
||||
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
if (
|
||||
_declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
|
||||
and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
|
||||
):
|
||||
return False
|
||||
|
||||
prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
|
||||
completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
|
||||
prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
|
||||
|
|
@ -338,13 +368,38 @@ def _transcription_usage_has_token_details(
|
|||
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
|
||||
|
||||
|
||||
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
|
||||
def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
|
||||
if pricing_model is None or custom_llm_provider is None:
|
||||
return False
|
||||
transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
|
||||
transcription_metadata: Final = (
|
||||
as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
|
||||
)
|
||||
return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
|
||||
|
||||
|
||||
def _transcription_duration_for_cost(
|
||||
completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
|
||||
) -> float:
|
||||
hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
|
||||
measured_duration: Final = normalized_audio_duration_seconds(
|
||||
hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
|
||||
)
|
||||
if measured_duration is not None and measured_duration > 0:
|
||||
return measured_duration
|
||||
provider_duration: Final = _get_transcription_usage_duration(completion_response)
|
||||
if (
|
||||
provider_duration is not None
|
||||
and provider_duration > 0
|
||||
and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
|
||||
):
|
||||
return provider_duration
|
||||
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
|
||||
return 0.0
|
||||
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
|
||||
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
|
||||
)
|
||||
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
|
||||
return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
|
||||
|
||||
|
||||
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
|
||||
|
|
@ -693,7 +748,7 @@ def cost_per_token(
|
|||
vertex_location=vertex_location,
|
||||
)
|
||||
elif call_type == "atranscription" or call_type == "transcription":
|
||||
if _transcription_usage_has_token_details(usage_block):
|
||||
if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
|
|
@ -1275,6 +1330,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
|
|||
)
|
||||
|
||||
|
||||
def _get_transcription_usage_duration(completion_response: object) -> float | None:
|
||||
response_fields: Final = as_str_mapping(completion_response)
|
||||
usage_object: Final = (
|
||||
response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
|
||||
)
|
||||
usage_fields: Final = as_str_mapping(usage_object)
|
||||
usage_type: Final = (
|
||||
usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
|
||||
)
|
||||
if usage_type != "duration":
|
||||
return None
|
||||
seconds: Final = (
|
||||
usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
|
||||
)
|
||||
return normalized_audio_duration_seconds(seconds)
|
||||
|
||||
|
||||
def _extraction_usage(usage: ExtractionUsage) -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=usage.input_tokens,
|
||||
|
|
@ -2000,20 +2072,8 @@ def completion_cost(
|
|||
prompt,
|
||||
)
|
||||
elif call_type in _TRANSCRIPTION_CALL_TYPES:
|
||||
# Check _hidden_params first (duration stored there to
|
||||
# avoid polluting the response body), then fall back to
|
||||
# the response attribute (for verbose_json responses that
|
||||
# naturally include duration from the provider).
|
||||
_hidden = getattr(completion_response, "_hidden_params", {}) or {}
|
||||
audio_transcription_file_duration = _hidden.get(
|
||||
"audio_transcription_duration",
|
||||
getattr(completion_response, "duration", 0.0),
|
||||
) or (
|
||||
completion_response.usage.seconds
|
||||
if isinstance(completion_response, TranscriptionResponse)
|
||||
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
|
||||
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
|
||||
else 0.0
|
||||
audio_transcription_file_duration = _transcription_duration_for_cost(
|
||||
completion_response, model, requested_model, custom_llm_provider
|
||||
)
|
||||
elif call_type in _RERANK_CALL_TYPES:
|
||||
if completion_response is not None and isinstance(completion_response, RerankResponse):
|
||||
|
|
@ -2123,6 +2183,7 @@ def completion_cost(
|
|||
litellm_logging_obj=litellm_logging_obj,
|
||||
custom_pricing_model=selected_model if custom_pricing else None,
|
||||
base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
|
||||
deployment_models=(base_model, requested_model),
|
||||
)
|
||||
elif call_type == _MCP_CALL_TYPE:
|
||||
from litellm.proxy.mcp_server.cost_calculator import (
|
||||
|
|
@ -3401,8 +3462,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
|
|||
def create_logging_realtime_object(
|
||||
usage: Usage, results: OpenAIRealtimeStreamList
|
||||
) -> LiteLLMRealtimeStreamLoggingObject:
|
||||
translated_input_seconds, _ = _translation_session_seconds(results)
|
||||
return LiteLLMRealtimeStreamLoggingObject(
|
||||
usage=usage,
|
||||
usage=usage.model_copy(update={"seconds": translated_input_seconds})
|
||||
if translated_input_seconds > 0
|
||||
else usage,
|
||||
results=results,
|
||||
)
|
||||
|
||||
|
|
@ -3487,6 +3551,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
|
|||
|
||||
|
||||
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
|
||||
_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
|
||||
|
||||
|
||||
class _LiveResponsePayload(BaseModel):
|
||||
|
|
@ -3709,6 +3774,7 @@ def handle_realtime_stream_cost_calculation(
|
|||
litellm_logging_obj: LitellmLoggingObject | None = None,
|
||||
custom_pricing_model: str | None = None,
|
||||
base_pricing_model: str | None = None,
|
||||
deployment_models: Sequence[str | None] = (),
|
||||
) -> float:
|
||||
"""
|
||||
Handles the cost calculation for realtime stream responses.
|
||||
|
|
@ -3753,6 +3819,13 @@ def handle_realtime_stream_cost_calculation(
|
|||
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
|
||||
else 0.0
|
||||
)
|
||||
translation_cost: Final = handle_realtime_translation_cost_calculation(
|
||||
results=results,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_model_name=litellm_model_name,
|
||||
deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
|
||||
session_model=received_model,
|
||||
)
|
||||
live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
|
||||
live_session_started: Final = any(result["type"] == "session.started" for result in results)
|
||||
live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
|
||||
|
|
@ -3809,12 +3882,19 @@ def handle_realtime_stream_cost_calculation(
|
|||
input_cost: Final = input_cost_per_token + live_response_input_cost
|
||||
output_cost: Final = output_cost_per_token + live_response_output_cost
|
||||
total_cost: Final = (
|
||||
input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
|
||||
input_cost
|
||||
+ output_cost
|
||||
+ transcription_cost
|
||||
+ translation_cost
|
||||
+ live_voice_cost
|
||||
+ audio_duration_cost
|
||||
+ text_message_cost
|
||||
)
|
||||
additional_costs: Final = tuple(
|
||||
(cost_name, cost)
|
||||
for cost_name, cost in (
|
||||
("transcription_cost", transcription_cost),
|
||||
("translation_cost", translation_cost),
|
||||
("live_voice_cost", live_voice_cost),
|
||||
("audio_duration_cost", audio_duration_cost),
|
||||
("text_message_cost", text_message_cost),
|
||||
|
|
@ -3835,6 +3915,55 @@ def handle_realtime_stream_cost_calculation(
|
|||
return total_cost
|
||||
|
||||
|
||||
def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
|
||||
usage_events: Final = tuple(
|
||||
usage
|
||||
for result in results
|
||||
if (event := as_str_mapping(result)) is not None
|
||||
and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
|
||||
and (usage := as_str_mapping(event.get("usage"))) is not None
|
||||
)
|
||||
input_seconds: Final = sum(
|
||||
seconds
|
||||
for usage in usage_events
|
||||
if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
|
||||
)
|
||||
output_seconds: Final = sum(
|
||||
output_duration
|
||||
for usage in usage_events
|
||||
if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
|
||||
)
|
||||
return float(input_seconds), float(output_seconds)
|
||||
|
||||
|
||||
def _translation_per_second_rates(
|
||||
models: Sequence[str | None], custom_llm_provider: str
|
||||
) -> tuple[float | None, float | None]:
|
||||
model_infos: Final = tuple(
|
||||
_lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
|
||||
)
|
||||
return (
|
||||
_first_declared_transcription_rate(("input_cost_per_second",), model_infos),
|
||||
_first_declared_transcription_rate(("output_cost_per_second",), model_infos),
|
||||
)
|
||||
|
||||
|
||||
def handle_realtime_translation_cost_calculation(
|
||||
results: OpenAIRealtimeStreamList,
|
||||
custom_llm_provider: str,
|
||||
litellm_model_name: str,
|
||||
deployment_pricing_models: Sequence[str | None] = (),
|
||||
session_model: str | None = None,
|
||||
) -> float:
|
||||
input_seconds, output_seconds = _translation_session_seconds(results)
|
||||
if input_seconds <= 0 and output_seconds <= 0:
|
||||
return 0.0
|
||||
input_rate, output_rate = _translation_per_second_rates(
|
||||
(*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
|
||||
)
|
||||
return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
|
||||
|
||||
|
||||
def handle_realtime_transcription_cost_calculation(
|
||||
results: OpenAIRealtimeStreamList,
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -3850,9 +3979,11 @@ def handle_realtime_transcription_cost_calculation(
|
|||
- {"type": "duration", "seconds": <float>} → priced via input_cost_per_second
|
||||
- {"type": "tokens", "input_tokens": ...} → priced via input/audio token cost
|
||||
"""
|
||||
completed_events: Final = [
|
||||
cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
|
||||
]
|
||||
completed_events: Final = tuple(
|
||||
event
|
||||
for result in results
|
||||
if (event := as_str_mapping(result)) is not None and event.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
|
||||
)
|
||||
if not completed_events:
|
||||
return 0.0
|
||||
|
||||
|
|
@ -3862,11 +3993,10 @@ def handle_realtime_transcription_cost_calculation(
|
|||
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
|
||||
)
|
||||
|
||||
total_cost = 0.0
|
||||
for event in completed_events:
|
||||
usage = event.get("usage") or {}
|
||||
total_cost += _transcription_usage_cost(usage, model_info, override_info)
|
||||
return total_cost
|
||||
return sum(
|
||||
_transcription_usage_cost(as_str_mapping(event.get("usage")) or {}, model_info, override_info)
|
||||
for event in completed_events
|
||||
)
|
||||
|
||||
|
||||
def _get_transcription_model_name_from_results(
|
||||
|
|
@ -3890,17 +4020,16 @@ def _get_transcription_model_name_from_results(
|
|||
return None
|
||||
|
||||
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
|
||||
]
|
||||
|
||||
|
||||
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
|
||||
def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
|
||||
match key:
|
||||
case "input_cost_per_second":
|
||||
return info.get("input_cost_per_second")
|
||||
case "output_cost_per_second":
|
||||
return info.get("output_cost_per_second")
|
||||
case "input_cost_per_audio_token":
|
||||
return info.get("input_cost_per_audio_token")
|
||||
case "output_cost_per_audio_token":
|
||||
return info.get("output_cost_per_audio_token")
|
||||
case "input_cost_per_token":
|
||||
return info.get("input_cost_per_token")
|
||||
case "output_cost_per_token":
|
||||
|
|
@ -3909,7 +4038,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
|
|||
assert_never(key)
|
||||
|
||||
|
||||
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
|
||||
def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
|
||||
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
|
||||
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
|
||||
if info is None:
|
||||
|
|
@ -3927,15 +4056,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
|
|||
)
|
||||
|
||||
|
||||
def _transcription_rate(
|
||||
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
|
||||
) -> float:
|
||||
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
|
||||
return next((rate for rate in rates if rate is not None), 0.0)
|
||||
def _first_declared_transcription_rate(
|
||||
keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
|
||||
) -> float | None:
|
||||
rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
|
||||
return next((rate for rate in rates if rate is not None), None)
|
||||
|
||||
|
||||
def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
|
||||
return _first_declared_transcription_rate(keys, infos) or 0.0
|
||||
|
||||
|
||||
def _transcription_usage_cost(
|
||||
usage: dict,
|
||||
usage: Mapping[str, object],
|
||||
model_info: ModelInfo | None,
|
||||
override_info: ModelInfo | None = None,
|
||||
) -> float:
|
||||
|
|
@ -3944,13 +4077,13 @@ def _transcription_usage_cost(
|
|||
|
||||
usage_type: Final = usage.get("type")
|
||||
if usage_type == "duration":
|
||||
seconds: Final = usage.get("seconds") or 0.0
|
||||
seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
|
||||
return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
|
||||
if usage_type == "tokens":
|
||||
input_token_details: Final = usage.get("input_token_details") or {}
|
||||
audio_tokens: Final = input_token_details.get("audio_tokens") or 0
|
||||
text_tokens: Final = input_token_details.get("text_tokens") or 0
|
||||
output_tokens: Final = usage.get("output_tokens") or 0
|
||||
input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
|
||||
audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
|
||||
text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
|
||||
output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
|
||||
audio_cost: Final = float(audio_tokens) * _transcription_rate(
|
||||
("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
|
|||
enrich_http_exception_with_guardrail_context(result, self)
|
||||
raise result
|
||||
|
||||
def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
|
||||
return (
|
||||
isinstance(request_data.get("guardrails"), list)
|
||||
and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
|
||||
)
|
||||
|
||||
async def async_post_call_success_deployment_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
|
|
@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
|
|||
"""
|
||||
Allow modifying / reviewing the response just after it's received from the deployment.
|
||||
"""
|
||||
# should run guardrail
|
||||
litellm_guardrails: Final = request_data.get("guardrails")
|
||||
if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
|
||||
return None
|
||||
|
||||
if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
|
||||
if not self.runs_post_call_deployment_hook(request_data):
|
||||
return None
|
||||
|
||||
target: Final = self._deployment_hook_target()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,322 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator, Callable, Iterator, Mapping
|
||||
from typing import Final, Protocol
|
||||
|
||||
import anyio
|
||||
from openai import AsyncStream, Stream
|
||||
from openai.types.audio import (
|
||||
TranscriptionStreamEvent,
|
||||
TranscriptionTextDeltaEvent,
|
||||
TranscriptionTextDoneEvent,
|
||||
)
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
||||
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
|
||||
TranscriptionUsageObjectTransformation,
|
||||
)
|
||||
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
|
||||
|
||||
TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed"
|
||||
|
||||
|
||||
_STREAM_FLAG: Final = TypeAdapter(bool)
|
||||
|
||||
|
||||
def transcription_stream_flag(value: object) -> bool | None:
|
||||
try:
|
||||
return _STREAM_FLAG.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def transcription_stream_unset(value: object) -> bool:
|
||||
return value is None or value == ""
|
||||
|
||||
|
||||
def transcription_stream_requested(value: object) -> bool:
|
||||
return transcription_stream_flag(value) is True
|
||||
|
||||
|
||||
TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
|
||||
|
||||
TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
|
||||
|
||||
|
||||
def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
|
||||
|
||||
model: Final = call_details.get("model")
|
||||
custom_llm_provider: Final = call_details.get("custom_llm_provider")
|
||||
extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
|
||||
|
||||
def map_exception(exception: Exception) -> Exception:
|
||||
try:
|
||||
mapped: Final[object] = exception_type(
|
||||
model=model if isinstance(model, str) else None,
|
||||
original_exception=exception,
|
||||
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
|
||||
extra_kwargs=extra_kwargs,
|
||||
)
|
||||
except Exception as mapped_exception: # noqa: BLE001 # exception_type raises the mapped exception
|
||||
return mapped_exception
|
||||
return mapped if isinstance(mapped, Exception) else exception
|
||||
|
||||
return map_exception
|
||||
|
||||
|
||||
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
|
||||
text: Final = response.text or ""
|
||||
usage: Final = response.usage
|
||||
languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
|
||||
done_event: Final = {
|
||||
"type": "transcript.text.done",
|
||||
"text": text,
|
||||
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
|
||||
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
|
||||
}
|
||||
return (
|
||||
*(
|
||||
f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
|
||||
for event in ({"type": "transcript.text.delta", "delta": text}, done_event)
|
||||
),
|
||||
TRANSCRIPTION_STREAM_DONE_FRAME,
|
||||
)
|
||||
|
||||
|
||||
class TranscriptionStreamLogging(Protocol):
|
||||
def success_handler(
|
||||
self,
|
||||
result: TranscriptionResponse,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None: ...
|
||||
|
||||
async def async_success_handler(
|
||||
self,
|
||||
result: TranscriptionResponse,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None: ...
|
||||
|
||||
def handle_sync_success_callbacks_for_async_calls(
|
||||
self,
|
||||
result: TranscriptionResponse,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None: ...
|
||||
|
||||
def failure_handler(
|
||||
self,
|
||||
exception: Exception,
|
||||
traceback_exception: str,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
*,
|
||||
for_async_request: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
async def async_failure_handler(
|
||||
self,
|
||||
exception: Exception,
|
||||
traceback_exception: str,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None: ...
|
||||
|
||||
def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
|
||||
|
||||
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
|
||||
|
||||
|
||||
class _TranscriptionEventCollector:
|
||||
def __init__(self, duration: float | None) -> None:
|
||||
self.duration = duration
|
||||
self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event
|
||||
self.done_event: TranscriptionTextDoneEvent | None = None
|
||||
|
||||
def add(self, event: TranscriptionStreamEvent) -> None:
|
||||
if isinstance(event, TranscriptionTextDeltaEvent):
|
||||
self.text_deltas.append(event.delta)
|
||||
elif isinstance(event, TranscriptionTextDoneEvent):
|
||||
self.done_event = event
|
||||
|
||||
def response(self) -> TranscriptionResponse:
|
||||
done_event: Final = self.done_event
|
||||
response: Final = TranscriptionResponse(
|
||||
text=done_event.text if done_event is not None else "".join(self.text_deltas),
|
||||
usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
|
||||
languages=getattr(done_event, "languages", None) if done_event is not None else None,
|
||||
)
|
||||
if self.duration is not None:
|
||||
response.set_audio_transcription_duration(self.duration)
|
||||
return response
|
||||
|
||||
|
||||
def _billed_usage(response: TranscriptionResponse) -> Usage:
|
||||
usage: Final[object] = getattr(response, "usage", None)
|
||||
token_usage: Final = (
|
||||
TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
|
||||
if isinstance(usage, TranscriptionUsageTokensObject)
|
||||
else None
|
||||
)
|
||||
return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
|
||||
|
||||
|
||||
def _record_billable_partial_spend(
|
||||
logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
|
||||
) -> None:
|
||||
partial_response: Final = collector.response()
|
||||
partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
|
||||
if partial_cost <= 0:
|
||||
return
|
||||
logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
|
||||
|
||||
|
||||
class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
|
||||
def __init__(
|
||||
self,
|
||||
stream: Stream[TranscriptionStreamEvent],
|
||||
logging_obj: TranscriptionStreamLogging,
|
||||
start_time: datetime.datetime,
|
||||
map_exception: TranscriptionStreamExceptionMapper,
|
||||
) -> None:
|
||||
self.__dict__.update(stream.__dict__)
|
||||
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
|
||||
"additional_headers": process_response_headers(stream.response.headers)
|
||||
}
|
||||
self._logging_obj = logging_obj
|
||||
self._start_time = start_time
|
||||
self._map_exception = map_exception
|
||||
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
|
||||
self._finalized = False
|
||||
self._failed = False
|
||||
source_iterator: Final = self._iterator
|
||||
self._iterator = self._logging_iterator(source_iterator)
|
||||
|
||||
def _logging_iterator(
|
||||
self, source_iterator: Iterator[TranscriptionStreamEvent]
|
||||
) -> Iterator[TranscriptionStreamEvent]:
|
||||
try:
|
||||
for event in source_iterator:
|
||||
self._collector.add(event)
|
||||
yield event
|
||||
except Exception as exception:
|
||||
self._failed = True
|
||||
_record_billable_partial_spend(self._logging_obj, self._collector)
|
||||
mapped_exception: Final = self._map_exception(exception)
|
||||
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
|
||||
self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
|
||||
raise mapped_exception from exception
|
||||
finally:
|
||||
self._finalize()
|
||||
|
||||
def _finalize(self) -> None:
|
||||
if self._finalized or self._failed:
|
||||
return
|
||||
self._finalized = True
|
||||
self._logging_obj.success_handler(
|
||||
self._collector.response(),
|
||||
self._start_time,
|
||||
datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
super().close()
|
||||
finally:
|
||||
self._finalize()
|
||||
|
||||
@property
|
||||
def logging_obj(self) -> TranscriptionStreamLogging:
|
||||
return self._logging_obj
|
||||
|
||||
|
||||
class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
|
||||
def __init__(
|
||||
self,
|
||||
stream: AsyncStream[TranscriptionStreamEvent],
|
||||
logging_obj: TranscriptionStreamLogging,
|
||||
start_time: datetime.datetime,
|
||||
map_exception: TranscriptionStreamExceptionMapper,
|
||||
) -> None:
|
||||
self.__dict__.update(stream.__dict__)
|
||||
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
|
||||
"additional_headers": process_response_headers(stream.response.headers)
|
||||
}
|
||||
self._logging_obj = logging_obj
|
||||
self._start_time = start_time
|
||||
self._map_exception = map_exception
|
||||
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
|
||||
self._finalized = False
|
||||
self._failed = False
|
||||
source_iterator: Final = self._iterator
|
||||
self._iterator = self._logging_iterator(source_iterator)
|
||||
|
||||
async def _logging_iterator(
|
||||
self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
|
||||
) -> AsyncIterator[TranscriptionStreamEvent]:
|
||||
try:
|
||||
async for event in source_iterator:
|
||||
self._collector.add(event)
|
||||
yield event
|
||||
except Exception as exception:
|
||||
self._failed = True
|
||||
_record_billable_partial_spend(self._logging_obj, self._collector)
|
||||
mapped_exception: Final = self._map_exception(exception)
|
||||
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
|
||||
self._logging_obj.failure_handler(
|
||||
mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
|
||||
)
|
||||
await self._logging_obj.async_failure_handler(
|
||||
mapped_exception,
|
||||
traceback.format_exc(),
|
||||
self._start_time,
|
||||
end_time,
|
||||
)
|
||||
raise mapped_exception from exception
|
||||
finally:
|
||||
await self._finalize()
|
||||
|
||||
async def _finalize(self) -> None:
|
||||
if self._finalized or self._failed:
|
||||
return
|
||||
self._finalized = True
|
||||
response: Final = self._collector.response()
|
||||
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
|
||||
self._logging_obj.handle_sync_success_callbacks_for_async_calls(
|
||||
result=response,
|
||||
start_time=self._start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
with anyio.CancelScope(shield=True):
|
||||
await self._logging_obj.async_success_handler(
|
||||
result=response,
|
||||
start_time=self._start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
try:
|
||||
await super().close()
|
||||
finally:
|
||||
await self._finalize()
|
||||
|
||||
@property
|
||||
def logging_obj(self) -> TranscriptionStreamLogging:
|
||||
return self._logging_obj
|
||||
|
||||
|
||||
def wrap_transcription_stream(
|
||||
stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
|
||||
logging_obj: TranscriptionStreamLogging,
|
||||
start_time: datetime.datetime,
|
||||
map_exception: TranscriptionStreamExceptionMapper,
|
||||
) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
|
||||
if isinstance(stream, AsyncStream):
|
||||
return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
|
||||
return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)
|
||||
|
|
@ -21,6 +21,16 @@ from litellm.types.files import (
|
|||
from litellm.types.utils import FileTypes
|
||||
|
||||
|
||||
def normalized_audio_duration_seconds(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return None
|
||||
try:
|
||||
seconds: Final = float(value)
|
||||
except OverflowError:
|
||||
return None
|
||||
return seconds if math.isfinite(seconds) and seconds >= 0 else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessedAudioFile:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ from types import MappingProxyType, TracebackType
|
|||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
|
||||
|
||||
from httpx import Response, Timeout
|
||||
from openai import AsyncStream as OpenAIAsyncStream
|
||||
from openai import Stream as OpenAIStream
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
|
|
@ -2128,6 +2130,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if resolved_cache_hit is True:
|
||||
return 0.0
|
||||
|
||||
if isinstance(result, (OpenAIStream, OpenAIAsyncStream)):
|
||||
return None
|
||||
|
||||
if is_unbilled_non_inference_call(
|
||||
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
|
||||
):
|
||||
|
|
@ -3606,6 +3611,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
|
||||
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
|
||||
return
|
||||
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
|
||||
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
@ -3620,7 +3626,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
try:
|
||||
## BUILD COMPLETE STREAMED RESPONSE
|
||||
complete_streaming_response: (
|
||||
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
|
||||
ModelResponse
|
||||
| TextCompletionResponse
|
||||
| ResponsesAPIResponse
|
||||
| InteractionsAPIResponse
|
||||
| TranscriptionResponse
|
||||
| None
|
||||
) = None
|
||||
if "complete_streaming_response" in self.model_call_details and not for_async_request:
|
||||
return # break out of this.
|
||||
|
|
@ -3636,7 +3647,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["complete_streaming_response"] = complete_streaming_response
|
||||
self._surface_response_headers_from_result(complete_streaming_response)
|
||||
self.model_call_details["response_cost"] = self.response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
result=complete_streaming_response if provider_transcription is None else provider_transcription
|
||||
)
|
||||
self._add_continued_partial_spend(complete_streaming_response)
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
|
|
@ -3648,7 +3659,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
## STANDARDIZED LOGGING PAYLOAD
|
||||
if not for_async_request:
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
logged_streaming_response, start_time, end_time
|
||||
logged_streaming_response if provider_transcription is None else provider_transcription,
|
||||
start_time,
|
||||
end_time,
|
||||
)
|
||||
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
|
||||
"standard_logging_object"
|
||||
|
|
@ -4175,6 +4188,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
|
||||
)
|
||||
)
|
||||
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
|
||||
start_time, end_time, result = self._success_handler_helper_fn(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
@ -4194,9 +4208,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
## BUILD COMPLETE STREAMED RESPONSE
|
||||
if "async_complete_streaming_response" in self.model_call_details:
|
||||
return # break out of this.
|
||||
complete_streaming_response: Final[
|
||||
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
|
||||
] = self._get_assembled_streaming_response(
|
||||
complete_streaming_response: Final = self._get_assembled_streaming_response(
|
||||
result=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
|
|
@ -4218,7 +4230,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
get_base_model_from_metadata(model_call_details=self.model_call_details)
|
||||
# base_model defaults to None if not set on model_info
|
||||
self.model_call_details["response_cost"] = self.response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
result=complete_streaming_response if provider_transcription is None else provider_transcription
|
||||
)
|
||||
|
||||
verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"])
|
||||
|
|
@ -4247,7 +4259,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
## STANDARDIZED LOGGING PAYLOAD
|
||||
try:
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
logged_streaming_response, start_time, end_time
|
||||
logged_streaming_response if provider_transcription is None else provider_transcription,
|
||||
start_time,
|
||||
end_time,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release)
|
||||
verbose_logger.exception(
|
||||
|
|
@ -5117,10 +5131,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
end_time: datetime.datetime,
|
||||
is_async: bool,
|
||||
streaming_chunks: list[object],
|
||||
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
|
||||
) -> (
|
||||
ModelResponse
|
||||
| TextCompletionResponse
|
||||
| ResponsesAPIResponse
|
||||
| InteractionsAPIResponse
|
||||
| TranscriptionResponse
|
||||
| None
|
||||
):
|
||||
if self.stream is not True:
|
||||
return None
|
||||
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
|
||||
if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)):
|
||||
return result
|
||||
elif isinstance(
|
||||
result,
|
||||
|
|
|
|||
|
|
@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost(
|
|||
class PromptTokensDetailsResult(TypedDict):
|
||||
cache_hit_tokens: int
|
||||
cache_hit_audio_tokens: ReadOnly[int]
|
||||
|
||||
cached_text_tokens: ReadOnly[int]
|
||||
cached_audio_tokens: ReadOnly[int]
|
||||
cached_image_tokens: ReadOnly[int]
|
||||
has_cached_tokens_details: ReadOnly[bool]
|
||||
cache_creation_tokens: int
|
||||
cache_creation_token_details: CacheCreationTokenDetails | None
|
||||
text_tokens: int
|
||||
|
|
@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
|
|||
return PromptTokensDetailsResult(
|
||||
cache_hit_tokens=cache_hit_tokens,
|
||||
cache_hit_audio_tokens=cached_audio_tokens,
|
||||
cached_text_tokens=cached_text_tokens,
|
||||
cached_audio_tokens=cached_audio_tokens,
|
||||
cached_image_tokens=cached_image_tokens,
|
||||
has_cached_tokens_details=cached_tokens_details is not None,
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
text_tokens=text_tokens,
|
||||
|
|
@ -1234,15 +1243,11 @@ def _calculate_input_cost(
|
|||
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
|
||||
|
||||
### CACHE READ COST - Now uses tiered pricing
|
||||
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
|
||||
audio_cache_read_rate: Final = get_cost_per_unit(
|
||||
model_info,
|
||||
get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
|
||||
None,
|
||||
)
|
||||
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
|
||||
prompt_cost += float(cache_hit_audio_tokens) * (
|
||||
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
|
||||
prompt_cost += _calculate_cache_read_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=model_info,
|
||||
cache_read_cost=cache_read_cost,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
### AUDIO COST
|
||||
|
|
@ -1331,6 +1336,38 @@ def _calculate_input_cost(
|
|||
return prompt_cost
|
||||
|
||||
|
||||
def _calculate_cache_read_cost(
|
||||
prompt_tokens_details: PromptTokensDetailsResult,
|
||||
model_info: ModelInfo,
|
||||
cache_read_cost: float,
|
||||
service_tier: str | None,
|
||||
) -> float:
|
||||
cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"]
|
||||
cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"]
|
||||
cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"]
|
||||
classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
|
||||
unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0)
|
||||
total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost
|
||||
float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost
|
||||
)
|
||||
|
||||
if cached_audio_tokens:
|
||||
cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier)
|
||||
cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost)
|
||||
total_cost += ( # rebind-ok: cached audio contributes to cache-read cost
|
||||
float(cached_audio_tokens) * float(cached_audio_cost or 0.0)
|
||||
)
|
||||
|
||||
if cached_image_tokens:
|
||||
cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier)
|
||||
cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost)
|
||||
total_cost += ( # rebind-ok: cached images contribute to cache-read cost
|
||||
float(cached_image_tokens) * float(cached_image_cost or 0.0)
|
||||
)
|
||||
|
||||
return total_cost
|
||||
|
||||
|
||||
def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
|
||||
"""
|
||||
Resolve the per-model regional-processing uplift multiplier for a given
|
||||
|
|
@ -1502,6 +1539,10 @@ def generic_cost_per_token(
|
|||
prompt_tokens_details = PromptTokensDetailsResult(
|
||||
cache_hit_tokens=0,
|
||||
cache_hit_audio_tokens=0,
|
||||
cached_text_tokens=0,
|
||||
cached_audio_tokens=0,
|
||||
cached_image_tokens=0,
|
||||
has_cached_tokens_details=False,
|
||||
cache_creation_tokens=0,
|
||||
cache_creation_token_details=None,
|
||||
text_tokens=usage.prompt_tokens,
|
||||
|
|
@ -1680,6 +1721,7 @@ class BilledTokenRates:
|
|||
cache_creation_input_token_cost: float
|
||||
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
|
||||
output_cost_per_reasoning_token: float
|
||||
cache_read_input_image_token_cost: float | None = None
|
||||
|
||||
def scaled(self, multiplier: float) -> "BilledTokenRates":
|
||||
if multiplier == 1.0:
|
||||
|
|
@ -1694,6 +1736,11 @@ class BilledTokenRates:
|
|||
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
|
||||
),
|
||||
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
|
||||
cache_read_input_image_token_cost=(
|
||||
self.cache_read_input_image_token_cost * multiplier
|
||||
if self.cache_read_input_image_token_cost is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1802,6 +1849,11 @@ def _cost_map_billed_rates(
|
|||
cache_creation_input_token_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
output_cost_per_reasoning_token=reasoning_rate,
|
||||
cache_read_input_image_token_cost=_get_cost_per_unit(
|
||||
model_info,
|
||||
_get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier),
|
||||
None,
|
||||
),
|
||||
).scaled(multiplier)
|
||||
|
||||
|
||||
|
|
@ -1882,6 +1934,12 @@ def get_token_type_cost_breakdown(
|
|||
cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
|
||||
usage
|
||||
)
|
||||
cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"]
|
||||
image_cache_read_rate: Final = (
|
||||
rates.cache_read_input_image_token_cost
|
||||
if rates.cache_read_input_image_token_cost is not None
|
||||
else rates.cache_read_input_token_cost
|
||||
)
|
||||
cache_creation_cost: Final = (
|
||||
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
|
||||
if custom_cost_per_token is not None
|
||||
|
|
@ -1895,8 +1953,9 @@ def get_token_type_cost_breakdown(
|
|||
return TokenTypeCostBreakdown(
|
||||
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
|
||||
cache_read_cost=(
|
||||
float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
|
||||
float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost
|
||||
+ float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
|
||||
+ float(cached_image_tokens) * image_cache_read_rate
|
||||
),
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
rates=rates,
|
||||
|
|
|
|||
|
|
@ -937,9 +937,11 @@ def convert_to_model_response_object(
|
|||
model_response_object.data = response_object["data"]
|
||||
|
||||
if "usage" in response_object and response_object["usage"] is not None:
|
||||
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
|
||||
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
|
||||
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
|
||||
embedding_usage: Final = model_response_object.usage or Usage()
|
||||
embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
|
||||
embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
|
||||
embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0)
|
||||
model_response_object.usage = embedding_usage
|
||||
|
||||
if start_time is not None and end_time is not None:
|
||||
model_response_object._response_ms = (
|
||||
|
|
@ -982,6 +984,10 @@ def convert_to_model_response_object(
|
|||
if key in response_object:
|
||||
setattr(model_response_object, key, response_object[key])
|
||||
|
||||
detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages"))
|
||||
if detected_languages is not None:
|
||||
model_response_object.languages = detected_languages
|
||||
|
||||
if "usage" in response_object and response_object["usage"] is not None:
|
||||
tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from collections.abc import Mapping
|
||||
from functools import lru_cache
|
||||
from typing import ClassVar, Final
|
||||
from typing import ClassVar, Final, get_args
|
||||
|
||||
from openai.types.chat.completion_create_params import (
|
||||
CompletionCreateParamsNonStreaming,
|
||||
|
|
@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.llms.anthropic import AnthropicMessagesRequest
|
||||
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
|
||||
from litellm.types.rerank import RerankRequest
|
||||
|
||||
|
||||
|
|
@ -178,6 +179,7 @@ class ModelParamHelper:
|
|||
|
||||
This follows the OpenAI API Spec
|
||||
"""
|
||||
litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams))
|
||||
try:
|
||||
from openai.types.audio.transcription_create_params import (
|
||||
TranscriptionCreateParamsNonStreaming,
|
||||
|
|
@ -187,11 +189,11 @@ class ModelParamHelper:
|
|||
non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys())
|
||||
streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys())
|
||||
|
||||
all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs)
|
||||
all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs)
|
||||
return all_transcription_kwargs
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Error getting transcription kwargs %s", str(e))
|
||||
return set()
|
||||
return litellm_transcription_kwargs
|
||||
|
||||
@staticmethod
|
||||
def _get_litellm_supported_responses_api_kwargs() -> set[str]:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
import traceback
|
||||
|
|
@ -8,7 +10,7 @@ from enum import Enum, auto
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
import litellm
|
||||
|
|
@ -22,6 +24,8 @@ from litellm.constants import (
|
|||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
|
||||
from litellm.llms.base_llm.realtime.transformation import (
|
||||
|
|
@ -35,6 +39,8 @@ from litellm.types.llms.openai import (
|
|||
OpenAIRealtimeResponseDelta,
|
||||
OpenAIRealtimeStreamResponseBaseObject,
|
||||
OpenAIRealtimeStreamSessionEvents,
|
||||
OpenAIRealtimeTranslationClosedEvent,
|
||||
OpenAIRealtimeTranslationDurationUsage,
|
||||
)
|
||||
from litellm.types.realtime import ALL_DELTA_TYPES
|
||||
|
||||
|
|
@ -42,9 +48,11 @@ from .litellm_logging import Logging as LiteLLMLogging
|
|||
from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
CLIENT_CONNECTION_CLASS = ClientConnection
|
||||
|
|
@ -96,6 +104,7 @@ class _ClientEventFrame(TypedDict, total=False):
|
|||
type: ReadOnly[str]
|
||||
item: ReadOnly[_ClientEventItem]
|
||||
session: ReadOnly[Mapping[str, object]]
|
||||
response: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
class _ResponseDoneBody(TypedDict, total=False):
|
||||
|
|
@ -212,6 +221,87 @@ def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, ob
|
|||
return _json_object(container.get(key))
|
||||
|
||||
|
||||
_BETA_AUDIO_FORMAT_BY_GA_TYPE: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"}
|
||||
)
|
||||
_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"output_modalities": "modalities", "max_output_tokens": "max_response_output_tokens"}
|
||||
)
|
||||
_GA_TO_BETA_AUDIO_INPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"format": "input_audio_format",
|
||||
"transcription": "input_audio_transcription",
|
||||
"noise_reduction": "input_audio_noise_reduction",
|
||||
"turn_detection": "turn_detection",
|
||||
}
|
||||
)
|
||||
_GA_TO_BETA_AUDIO_OUTPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"format": "output_audio_format", "voice": "voice", "speed": "speed"}
|
||||
)
|
||||
_GA_ONLY_SESSION_FIELDS: Final = frozenset(("type", "audio", *_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS))
|
||||
|
||||
|
||||
def _beta_session_value(ga_field: str, value: object) -> object:
|
||||
if ga_field == "output_modalities":
|
||||
return ["audio", "text"] if value == ["audio"] else value
|
||||
if ga_field != "format":
|
||||
return value
|
||||
ga_format: Final = _json_object(value)
|
||||
format_type: Final = None if ga_format is None else ga_format.get("type")
|
||||
return _BETA_AUDIO_FORMAT_BY_GA_TYPE.get(format_type, value) if isinstance(format_type, str) else value
|
||||
|
||||
|
||||
def _beta_session_fields(ga_fields: Mapping[str, object], beta_names: Mapping[str, str]) -> Mapping[str, object]:
|
||||
return {
|
||||
beta_names[name]: _beta_session_value(name, value) for name, value in ga_fields.items() if name in beta_names
|
||||
}
|
||||
|
||||
|
||||
def ga_session_to_beta(session: Mapping[str, object]) -> Mapping[str, object]:
|
||||
audio: Final = _mapping_field(session, "audio") or _EMPTY_SESSION
|
||||
return {
|
||||
**{key: value for key, value in session.items() if key not in _GA_ONLY_SESSION_FIELDS},
|
||||
**_beta_session_fields(session, _GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS),
|
||||
**_beta_session_fields(_mapping_field(audio, "input") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_INPUT_FIELDS),
|
||||
**_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS),
|
||||
}
|
||||
|
||||
|
||||
_GA_ONLY_RESPONSE_FIELDS: Final = frozenset(("audio", "output_modalities"))
|
||||
_GA_TO_BETA_RESPONSE_FIELDS: Final[Mapping[str, str]] = MappingProxyType({"output_modalities": "modalities"})
|
||||
|
||||
|
||||
def ga_response_to_beta(response: Mapping[str, object]) -> Mapping[str, object]:
|
||||
audio: Final = _mapping_field(response, "audio") or _EMPTY_SESSION
|
||||
return {
|
||||
**{key: value for key, value in response.items() if key not in _GA_ONLY_RESPONSE_FIELDS},
|
||||
**_beta_session_fields(response, _GA_TO_BETA_RESPONSE_FIELDS),
|
||||
**_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS),
|
||||
}
|
||||
|
||||
|
||||
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
|
||||
_GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"text": "output_text", "audio": "output_audio"}
|
||||
)
|
||||
|
||||
|
||||
def _ga_assistant_content_block(block: object) -> object:
|
||||
content_block: Final = _json_object(block)
|
||||
block_type: Final = None if content_block is None else content_block.get("type")
|
||||
if content_block is None or not isinstance(block_type, str):
|
||||
return block
|
||||
ga_type: Final = _GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE.get(block_type)
|
||||
return block if ga_type is None else {**content_block, "type": ga_type}
|
||||
|
||||
|
||||
def beta_item_to_ga(item: Mapping[str, object]) -> Mapping[str, object]:
|
||||
content: Final = item.get("content")
|
||||
if item.get("role") != "assistant" or not isinstance(content, list):
|
||||
return item
|
||||
return {**item, "content": [_ga_assistant_content_block(block) for block in _JSON_ARRAY.validate_python(content)]}
|
||||
|
||||
|
||||
def _guardrail_event_id() -> str:
|
||||
return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
|
||||
|
||||
|
|
@ -270,6 +360,26 @@ def _gate_session_update(
|
|||
return {**client_session, "turn_detection": gate(None)}
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _translation_audio_format_adapter() -> "TypeAdapter[str | RealtimeAudioFormats | None]":
|
||||
from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats
|
||||
|
||||
return TypeAdapter(str | RealtimeAudioFormats | None)
|
||||
|
||||
|
||||
def _translation_audio_bytes_per_second(audio_format: "str | RealtimeAudioFormats | None") -> float | None:
|
||||
from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU
|
||||
|
||||
match audio_format:
|
||||
case "pcm16" | AudioPCM():
|
||||
return 48000.0
|
||||
case "g711_ulaw" | "g711_alaw" | AudioPCMU() | AudioPCMA():
|
||||
return 8000.0
|
||||
case _:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class RealtimeEventNormalizer(Protocol):
|
||||
def should_drop(self, event: object) -> bool: ...
|
||||
def normalize(self, event: dict) -> dict: ...
|
||||
|
|
@ -294,6 +404,79 @@ DefaultLoggedRealTimeEventTypes: Final = [
|
|||
]
|
||||
|
||||
|
||||
TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE: Final = "Translation sessions cannot enforce configured realtime guardrails"
|
||||
|
||||
|
||||
def _guardrails_run_for_event_hooks(
|
||||
request_data: Mapping[str, object],
|
||||
event_hooks: Sequence["GuardrailEventHooks"],
|
||||
callbacks: Sequence[object],
|
||||
) -> bool:
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
|
||||
return any(
|
||||
isinstance(cb, CustomGuardrail)
|
||||
and any(cb.should_run_guardrail(data={**request_data, "stream": True}, event_type=et) for et in event_hooks)
|
||||
for cb in callbacks
|
||||
)
|
||||
|
||||
|
||||
def _opted_out_of_always_on_guardrail(callback: object, request_data: Mapping[str, object]) -> bool:
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
|
||||
return (
|
||||
isinstance(callback, CustomGuardrail)
|
||||
and callback.event_hook is None
|
||||
and callback.get_disable_global_guardrail(dict(request_data)) is True
|
||||
)
|
||||
|
||||
|
||||
def _realtime_runner_guardrail(
|
||||
callback: object,
|
||||
check_data: Mapping[str, object],
|
||||
event_hooks: Sequence["GuardrailEventHooks"],
|
||||
) -> "CustomGuardrail | None":
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
|
||||
if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks:
|
||||
return None
|
||||
runs: Final = any(
|
||||
callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks
|
||||
)
|
||||
return callback if runs else None
|
||||
|
||||
|
||||
def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool:
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
callbacks: Final = tuple(
|
||||
callback for callback in litellm.callbacks if not _opted_out_of_always_on_guardrail(callback, request_data)
|
||||
)
|
||||
if _guardrails_run_for_event_hooks(request_data, (GuardrailEventHooks.post_call,), callbacks):
|
||||
return True
|
||||
if any(
|
||||
isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket
|
||||
for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata"))
|
||||
):
|
||||
from litellm.proxy.utils import pipeline_managed_guardrail_names
|
||||
|
||||
if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names(
|
||||
request_data, "post_call"
|
||||
):
|
||||
return True
|
||||
return realtime_guardrails_apply(request_data, callbacks)
|
||||
|
||||
|
||||
def realtime_guardrails_apply(request_data: Mapping[str, object], callbacks: Sequence[object]) -> bool:
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
check_data: Final = {**request_data, "stream": True}
|
||||
runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call)
|
||||
return any(
|
||||
_realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks
|
||||
)
|
||||
|
||||
|
||||
class RealTimeStreaming:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -303,12 +486,15 @@ class RealTimeStreaming:
|
|||
provider_config: BaseRealtimeConfig | None = None,
|
||||
model: str = "",
|
||||
user_api_key_dict: object | None = None,
|
||||
request_data: dict | None = None,
|
||||
request_data: Mapping[str, object] | None = None,
|
||||
backend_uses_beta_protocol: bool | None = None,
|
||||
force_transcription_model: str | None = None,
|
||||
event_normalizer: RealtimeEventNormalizer | None = None,
|
||||
usage_meter: RealtimeUsageMeter | None = None,
|
||||
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
|
||||
translation_session: bool = False,
|
||||
bridges_beta_client_to_ga: bool = False,
|
||||
drop_params: bool = False,
|
||||
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
|
|
@ -326,14 +512,24 @@ class RealTimeStreaming:
|
|||
self._logging_worker = logging_worker
|
||||
self.messages: list[OpenAIRealtimeEvents] = []
|
||||
self._backend_sent_frames: bool = False
|
||||
self._backend_sent_non_error_frames: bool = False
|
||||
self.input_message: dict = {}
|
||||
self.input_messages: list[dict[str, str]] = []
|
||||
self.session_tools: list[dict] = []
|
||||
self.tool_calls: list[dict] = []
|
||||
self._is_translation_session = translation_session
|
||||
self._bridges_beta_client_to_ga: Final = bridges_beta_client_to_ga
|
||||
self._remaps_all_beta_fields_to_ga: Final = bridges_beta_client_to_ga or translation_session
|
||||
self._drop_params: Final = drop_params
|
||||
self._translation_input_seconds = 0.0
|
||||
self._translation_input_bytes_per_second = 48000.0
|
||||
self._translation_output_seconds = 0.0
|
||||
self._translation_output_bytes_per_second = 48000.0
|
||||
self._translation_usage_finalized = False
|
||||
|
||||
# Detect whether the client is explicitly opting into the beta protocol.
|
||||
self._client_wants_beta = self.detect_beta_header(websocket)
|
||||
self._backend_uses_beta_protocol = (
|
||||
self._backend_uses_beta_protocol = not translation_session and (
|
||||
self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol
|
||||
)
|
||||
|
||||
|
|
@ -352,7 +548,7 @@ class RealTimeStreaming:
|
|||
self.current_delta_type: ALL_DELTA_TYPES | None = None
|
||||
self.session_configuration_request: str | None = None
|
||||
self.user_api_key_dict = user_api_key_dict
|
||||
self.request_data: dict = request_data or {}
|
||||
self.request_data: Mapping[str, object] = request_data or {}
|
||||
# Violation counter for end_session_after_n_fails support
|
||||
self._violation_count: int = 0
|
||||
# When a text message is blocked, hold the guardrail reason so the next
|
||||
|
|
@ -384,6 +580,7 @@ class RealTimeStreaming:
|
|||
# their input_audio_transcription.completed usage drives duration-based cost.
|
||||
self._force_transcription_model = force_transcription_model
|
||||
self._is_transcription_session: bool = force_transcription_model is not None
|
||||
self._bound_nested_transcription_model: str | None = None
|
||||
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
|
||||
self._event_normalizer = event_normalizer
|
||||
self._gate_turn_detection: Final[_TurnDetectionGate] = (
|
||||
|
|
@ -407,6 +604,7 @@ class RealTimeStreaming:
|
|||
_MAX_BUFFERED_BYTES: int = 10 * 1024 * 1024 # 10 MB
|
||||
|
||||
_SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"])
|
||||
_RESPONSE_EVENT_TYPES: Final = frozenset(("response.created", "response.done"))
|
||||
_CLIENT_AUDIO_BUFFER_TYPES = frozenset(
|
||||
[
|
||||
"input_audio_buffer.append",
|
||||
|
|
@ -429,8 +627,8 @@ class RealTimeStreaming:
|
|||
_DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"])
|
||||
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
|
||||
"pcm16": {"type": "audio/pcm", "rate": 24000},
|
||||
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
|
||||
"g711_alaw": {"type": "audio/G711-alaw", "rate": 8000},
|
||||
"g711_ulaw": {"type": "audio/pcmu"},
|
||||
"g711_alaw": {"type": "audio/pcma"},
|
||||
}
|
||||
# GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1)
|
||||
_GA_TO_BETA_EVENT_TYPES: dict[str, str] = {
|
||||
|
|
@ -449,12 +647,12 @@ class RealTimeStreaming:
|
|||
|
||||
def _should_store_message(
|
||||
self,
|
||||
message_obj: dict | OpenAIRealtimeEvents,
|
||||
message_obj: Mapping[str, object],
|
||||
) -> bool:
|
||||
_msg_type: Final = message_obj["type"] if "type" in message_obj else None
|
||||
if self.logged_real_time_event_types == "*":
|
||||
return True
|
||||
if _msg_type and _msg_type in self.logged_real_time_event_types:
|
||||
if isinstance(_msg_type, str) and _msg_type in self.logged_real_time_event_types:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
@ -555,7 +753,7 @@ class RealTimeStreaming:
|
|||
input_messages, so it is not duplicated into the response log here.
|
||||
"""
|
||||
try:
|
||||
usage: Final = event_obj.get("usage")
|
||||
usage: Final = as_str_mapping(event_obj.get("usage"))
|
||||
if usage is None:
|
||||
return
|
||||
# If this event type is already captured by store_message (e.g. the user
|
||||
|
|
@ -623,6 +821,7 @@ class RealTimeStreaming:
|
|||
|
||||
async def log_messages(self):
|
||||
"""Log messages in list"""
|
||||
self._finalize_translation_usage()
|
||||
if self.logging_obj:
|
||||
if self._usage_meter is not None:
|
||||
self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot()
|
||||
|
|
@ -641,6 +840,142 @@ class RealTimeStreaming:
|
|||
)
|
||||
self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
|
||||
|
||||
def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None:
|
||||
if not self._is_translation_session:
|
||||
return
|
||||
if event_obj.get("type") == "session.closed":
|
||||
usage: Final = as_str_mapping(event_obj.get("usage"))
|
||||
output_seconds: Final = (
|
||||
normalized_audio_duration_seconds(usage.get("output_seconds")) if usage is not None else None
|
||||
)
|
||||
reported_input_seconds: Final = (
|
||||
normalized_audio_duration_seconds(usage.get("input_seconds")) if usage is not None else None
|
||||
)
|
||||
input_seconds: Final = (
|
||||
reported_input_seconds
|
||||
if reported_input_seconds is not None
|
||||
else self._translation_input_seconds or None
|
||||
)
|
||||
synthetic_output_seconds: Final = self._translation_output_seconds or None
|
||||
resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds
|
||||
if input_seconds is not None or resolved_output_seconds is not None:
|
||||
if self._should_store_message(event_obj):
|
||||
supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage(
|
||||
type="duration",
|
||||
input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0,
|
||||
output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0,
|
||||
)
|
||||
if (
|
||||
supplemental_usage.get("input_seconds", 0.0) > 0
|
||||
or supplemental_usage.get("output_seconds", 0.0) > 0
|
||||
):
|
||||
self.messages.append(
|
||||
OpenAIRealtimeTranslationClosedEvent(
|
||||
type="session.closed",
|
||||
usage=supplemental_usage,
|
||||
)
|
||||
)
|
||||
else:
|
||||
normalized_usage: Final = (
|
||||
OpenAIRealtimeTranslationDurationUsage(
|
||||
type="duration",
|
||||
input_seconds=input_seconds,
|
||||
output_seconds=float(resolved_output_seconds or 0.0),
|
||||
)
|
||||
if input_seconds is not None
|
||||
else OpenAIRealtimeTranslationDurationUsage(
|
||||
type="duration", output_seconds=float(resolved_output_seconds or 0.0)
|
||||
)
|
||||
)
|
||||
self.messages.append(
|
||||
OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage)
|
||||
)
|
||||
self._translation_usage_finalized = True
|
||||
return
|
||||
self._capture_translation_audio_formats(event_obj)
|
||||
if event_obj.get("type") not in (
|
||||
"session.output_audio.delta",
|
||||
"response.output_audio.delta",
|
||||
"response.audio.delta",
|
||||
):
|
||||
return
|
||||
delta: Final = event_obj.get("delta")
|
||||
if not isinstance(delta, str):
|
||||
return
|
||||
try:
|
||||
decoded: Final = base64.b64decode(delta, validate=True)
|
||||
except (ValueError, TypeError):
|
||||
return
|
||||
self._translation_output_seconds += len(decoded) / self._translation_output_bytes_per_second
|
||||
|
||||
def _capture_translation_input_audio(self, message: str) -> None:
|
||||
if not self._is_translation_session:
|
||||
return
|
||||
try:
|
||||
event: Final = _decode_json_object(message)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return
|
||||
if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str):
|
||||
return
|
||||
try:
|
||||
decoded: Final = base64.b64decode(audio, validate=True)
|
||||
except (ValueError, TypeError):
|
||||
return
|
||||
self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second
|
||||
|
||||
def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None:
|
||||
if event_obj.get("type") not in ("session.created", "session.updated"):
|
||||
return
|
||||
session: Final = as_str_mapping(event_obj.get("session"))
|
||||
if session is None:
|
||||
return
|
||||
audio: Final = as_str_mapping(session.get("audio"))
|
||||
if audio is None:
|
||||
return
|
||||
for direction, raw_configuration in (("input", audio.get("input")), ("output", audio.get("output"))):
|
||||
if (configuration := as_str_mapping(raw_configuration)) is None:
|
||||
continue
|
||||
try:
|
||||
if (
|
||||
bytes_per_second := _translation_audio_bytes_per_second(
|
||||
_translation_audio_format_adapter().validate_python(configuration.get("format"))
|
||||
)
|
||||
) is None:
|
||||
continue
|
||||
except ValidationError:
|
||||
continue
|
||||
if direction == "input":
|
||||
self._translation_input_bytes_per_second = bytes_per_second
|
||||
else:
|
||||
self._translation_output_bytes_per_second = bytes_per_second
|
||||
|
||||
def _finalize_translation_usage(self) -> None:
|
||||
if self._translation_usage_finalized:
|
||||
return
|
||||
for event in self.messages:
|
||||
if event.get("type") != "session.closed":
|
||||
continue
|
||||
if (event_usage := as_str_mapping(event.get("usage"))) is not None and normalized_audio_duration_seconds(
|
||||
event_usage.get("output_seconds")
|
||||
) is not None:
|
||||
self._translation_usage_finalized = True
|
||||
return
|
||||
if self._translation_output_seconds == 0 and self._translation_input_seconds == 0:
|
||||
return
|
||||
synthetic_usage: Final = (
|
||||
OpenAIRealtimeTranslationDurationUsage(
|
||||
type="duration",
|
||||
input_seconds=self._translation_input_seconds,
|
||||
output_seconds=self._translation_output_seconds,
|
||||
)
|
||||
if self._translation_input_seconds > 0
|
||||
else OpenAIRealtimeTranslationDurationUsage(
|
||||
type="duration", output_seconds=self._translation_output_seconds
|
||||
)
|
||||
)
|
||||
self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage))
|
||||
self._translation_usage_finalized = True
|
||||
|
||||
async def _send_to_backend(self, message: str) -> bool:
|
||||
"""Send a message to the backend WebSocket.
|
||||
|
||||
|
|
@ -653,10 +988,25 @@ class RealTimeStreaming:
|
|||
backend, False if the provider transformation produced no output and
|
||||
the message was effectively dropped.
|
||||
"""
|
||||
message = self._enforce_transcription_session_model(message)
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
||||
try:
|
||||
authorized_message: Final = await self._apply_nested_transcription_model_policy(message)
|
||||
except ProxyException as denied:
|
||||
denial_message: Final = redact_internal_details_from_client_message(denied.message)
|
||||
try:
|
||||
await self.websocket.send_text(realtime_error_event(denial_message, error_type="permission_error"))
|
||||
finally:
|
||||
await self.websocket.close(
|
||||
code=1008,
|
||||
reason=websocket_close_reason(
|
||||
denial_message, fallback="Nested transcription model is not authorized"
|
||||
),
|
||||
)
|
||||
return False
|
||||
if self.provider_config:
|
||||
transformed: Final = self.provider_config.transform_realtime_request(
|
||||
message, self.model, self.session_configuration_request
|
||||
authorized_message, self.model, self.session_configuration_request
|
||||
)
|
||||
sent = False
|
||||
for msg in transformed:
|
||||
|
|
@ -691,14 +1041,94 @@ class RealTimeStreaming:
|
|||
if is_content_message:
|
||||
self._content_sent_after_setup = True
|
||||
sent = True
|
||||
if sent:
|
||||
self._capture_translation_input_audio(authorized_message)
|
||||
return sent
|
||||
await self.backend_ws.send(message)
|
||||
await self.backend_ws.send(authorized_message)
|
||||
if self._usage_meter is not None:
|
||||
sent_event: Final = self._parse_backend_event(message)
|
||||
sent_event: Final = self._parse_backend_event(authorized_message)
|
||||
if sent_event is not None:
|
||||
self._usage_meter.observe_client_event(sent_event)
|
||||
self._capture_translation_input_audio(authorized_message)
|
||||
return True
|
||||
|
||||
async def _apply_nested_transcription_model_policy(self, message: str) -> str:
|
||||
if self._force_transcription_model is not None:
|
||||
return self._enforce_transcription_session_model(message)
|
||||
if self._is_translation_session:
|
||||
return await self._enforce_translation_nested_transcription_model(message)
|
||||
return message
|
||||
|
||||
def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None:
|
||||
try:
|
||||
message_obj: Final = _decode_json_object(message)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
if message_obj.get("type") not in (
|
||||
"session.update",
|
||||
"transcription_session.update",
|
||||
):
|
||||
return None
|
||||
return message_obj
|
||||
|
||||
def _nested_transcription_models_from_session(
|
||||
self,
|
||||
session: Mapping[str, object],
|
||||
) -> tuple[str, ...]:
|
||||
audio: Final = as_str_mapping(session.get("audio"))
|
||||
audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None
|
||||
nested_transcription: Final = audio_input.get("transcription") if audio_input is not None else None
|
||||
nested_model: Final = self._transcription_model_value(nested_transcription)
|
||||
flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription"))
|
||||
return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None))
|
||||
|
||||
def _transcription_model_value(self, transcription_config: object) -> str | None:
|
||||
if (fields := as_str_mapping(transcription_config)) is None:
|
||||
return None
|
||||
model: Final = fields.get("model")
|
||||
if isinstance(model, str) and model:
|
||||
return model
|
||||
return None
|
||||
|
||||
def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str:
|
||||
message_obj: Final = self._session_update_message_obj(message)
|
||||
if message_obj is None:
|
||||
return message
|
||||
session: Final = as_str_mapping(message_obj.get("session"))
|
||||
if session is None:
|
||||
return message
|
||||
transcription: Final = as_str_mapping(session.get("input_audio_transcription"))
|
||||
rewrite_flat: Final = transcription is not None and transcription.get("model") != authorized_model
|
||||
audio: Final = as_str_mapping(session.get("audio"))
|
||||
audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None
|
||||
nested_transcription: Final = (
|
||||
as_str_mapping(audio_input.get("transcription")) if audio_input is not None else None
|
||||
)
|
||||
rewrite_nested: Final = (
|
||||
nested_transcription is not None and nested_transcription.get("model") != authorized_model
|
||||
)
|
||||
if not rewrite_flat and not rewrite_nested:
|
||||
return message
|
||||
updated_session: Final = {
|
||||
**session,
|
||||
**(
|
||||
{"input_audio_transcription": {**transcription, "model": authorized_model}}
|
||||
if transcription is not None and rewrite_flat
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
{
|
||||
"audio": {
|
||||
**audio,
|
||||
"input": {**audio_input, "transcription": {**nested_transcription, "model": authorized_model}},
|
||||
}
|
||||
}
|
||||
if audio is not None and audio_input is not None and nested_transcription is not None and rewrite_nested
|
||||
else {}
|
||||
),
|
||||
}
|
||||
return json.dumps({**message_obj, "session": updated_session})
|
||||
|
||||
def _builds_setup_lazily(self) -> bool:
|
||||
if self.provider_config is None:
|
||||
return False
|
||||
|
|
@ -877,56 +1307,52 @@ class RealTimeStreaming:
|
|||
if self._force_transcription_model is None:
|
||||
return message
|
||||
|
||||
try:
|
||||
message_obj: Final = _decode_json_object(message)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
message_obj: Final = self._session_update_message_obj(message)
|
||||
if message_obj is None:
|
||||
return message
|
||||
|
||||
if message_obj.get("type") not in (
|
||||
"session.update",
|
||||
"transcription_session.update",
|
||||
):
|
||||
return message
|
||||
|
||||
session: Final = message_obj.get("session")
|
||||
if not isinstance(session, dict):
|
||||
return message
|
||||
|
||||
if session.get("type") == "transcription":
|
||||
session: Final = as_str_mapping(message_obj.get("session"))
|
||||
if session is not None and session.get("type") == "transcription":
|
||||
self._is_transcription_session = True
|
||||
return self._rewrite_session_update_transcription_model(message, self._force_transcription_model)
|
||||
|
||||
authorized_model: Final = self._force_transcription_model
|
||||
changed = False
|
||||
async def _enforce_translation_nested_transcription_model(self, message: str) -> str:
|
||||
message_obj: Final = self._session_update_message_obj(message)
|
||||
session: Final = as_str_mapping(message_obj.get("session")) if message_obj is not None else None
|
||||
bound_model: Final = self._bound_nested_transcription_model
|
||||
requested_models: Final = (
|
||||
tuple(model for model in self._nested_transcription_models_from_session(session) if model != bound_model)
|
||||
if session is not None
|
||||
else ()
|
||||
)
|
||||
if not requested_models:
|
||||
return (
|
||||
message
|
||||
if bound_model is None
|
||||
else self._rewrite_session_update_transcription_model(message, bound_model)
|
||||
)
|
||||
|
||||
transcription: Final = session.get("input_audio_transcription")
|
||||
if isinstance(transcription, dict) and transcription.get("model") != authorized_model:
|
||||
session["input_audio_transcription"] = {
|
||||
**transcription,
|
||||
"model": authorized_model,
|
||||
}
|
||||
changed = True
|
||||
valid_token: Final = self.user_api_key_dict
|
||||
|
||||
audio: Final = session.get("audio")
|
||||
if isinstance(audio, dict):
|
||||
audio_input: Final = audio.get("input")
|
||||
if isinstance(audio_input, dict):
|
||||
nested_transcription: Final = audio_input.get("transcription")
|
||||
if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model:
|
||||
session["audio"] = {
|
||||
**audio,
|
||||
"input": {
|
||||
**audio_input,
|
||||
"transcription": {
|
||||
**nested_transcription,
|
||||
"model": authorized_model,
|
||||
},
|
||||
},
|
||||
}
|
||||
changed = True
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
|
||||
from litellm.proxy.proxy_server import llm_model_list, llm_router
|
||||
from litellm.proxy.realtime_endpoints.endpoints import nested_transcription_upstream_model
|
||||
|
||||
if not changed:
|
||||
if not isinstance(valid_token, UserAPIKeyAuth):
|
||||
return message
|
||||
return json.dumps(message_obj)
|
||||
|
||||
for requested_model in requested_models:
|
||||
await can_key_call_resolved_model(
|
||||
model=requested_model,
|
||||
valid_token=valid_token,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
upstream_model: Final = nested_transcription_upstream_model(
|
||||
requested_models[0], valid_token.team_id, llm_router
|
||||
)
|
||||
self._bound_nested_transcription_model = upstream_model
|
||||
return self._rewrite_session_update_transcription_model(message, upstream_model)
|
||||
|
||||
def _uses_deferred_backend_setup(self) -> bool:
|
||||
"""True when setup is deferred until the client's first session.update."""
|
||||
|
|
@ -1047,7 +1473,19 @@ class RealTimeStreaming:
|
|||
return event
|
||||
|
||||
def _event_to_client_json(self, event: dict) -> str:
|
||||
return json.dumps(self._normalize_event_for_ga_client(event))
|
||||
return json.dumps(self._session_event_for_client(self._normalize_event_for_ga_client(event)))
|
||||
|
||||
def _session_event_for_client(self, event: Mapping[str, object]) -> Mapping[str, object]:
|
||||
if not self._bridges_beta_client_to_ga:
|
||||
return event
|
||||
event_type: Final = event.get("type")
|
||||
if event_type in self._SESSION_EVENT_TYPES:
|
||||
session: Final = _mapping_field(event, "session")
|
||||
return event if session is None else {**event, "session": ga_session_to_beta(session)}
|
||||
if event_type in self._RESPONSE_EVENT_TYPES:
|
||||
response: Final = _mapping_field(event, "response")
|
||||
return event if response is None else {**event, "response": ga_response_to_beta(response)}
|
||||
return event
|
||||
|
||||
async def _send_event_to_client(self, event: object, event_str: str) -> bool:
|
||||
if self._should_drop_event_from_client(event):
|
||||
|
|
@ -1060,7 +1498,7 @@ class RealTimeStreaming:
|
|||
translated: Final = self._translate_event_to_beta(event)
|
||||
if translated is None:
|
||||
return False
|
||||
await self.websocket.send_text(json.dumps(translated))
|
||||
await self.websocket.send_text(json.dumps(self._session_event_for_client(translated)))
|
||||
return True
|
||||
except Exception as e:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -1183,32 +1621,7 @@ class RealTimeStreaming:
|
|||
self,
|
||||
event_hooks: Sequence["GuardrailEventHooks"],
|
||||
) -> bool:
|
||||
"""Return True if any callback would run for one of ``event_hooks``."""
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
|
||||
return any(
|
||||
isinstance(cb, CustomGuardrail)
|
||||
and any(
|
||||
cb.should_run_guardrail(
|
||||
data={**self.request_data, "stream": True},
|
||||
event_type=et,
|
||||
)
|
||||
for et in event_hooks
|
||||
)
|
||||
for cb in litellm.callbacks
|
||||
)
|
||||
|
||||
def _has_realtime_guardrails(self) -> bool:
|
||||
"""Return True if any callback is registered for realtime guardrail event types."""
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
return self._has_realtime_guardrails_for_event_hooks(
|
||||
[
|
||||
GuardrailEventHooks.realtime_input_transcription,
|
||||
GuardrailEventHooks.pre_call,
|
||||
GuardrailEventHooks.post_call,
|
||||
]
|
||||
)
|
||||
return _guardrails_run_for_event_hooks(self.request_data, event_hooks, litellm.callbacks)
|
||||
|
||||
def _has_audio_transcription_guardrails(self) -> bool:
|
||||
"""Return True when a guardrail is configured for the audio/VAD transcript path.
|
||||
|
|
@ -1222,8 +1635,12 @@ class RealTimeStreaming:
|
|||
return self._has_realtime_guardrails_for_event_hooks([GuardrailEventHooks.realtime_input_transcription])
|
||||
|
||||
def _should_disable_vad_auto_response(self) -> bool:
|
||||
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
|
||||
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
|
||||
"""Transcription and translation sessions have no assistant turn to gate and reject realtime session updates."""
|
||||
return (
|
||||
not self._is_transcription_session
|
||||
and not self._is_translation_session
|
||||
and self._has_audio_transcription_guardrails()
|
||||
)
|
||||
|
||||
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
|
||||
return (
|
||||
|
|
@ -1264,7 +1681,6 @@ class RealTimeStreaming:
|
|||
transcript completion uses ``realtime_input_transcription`` only;
|
||||
typed user messages and tool outputs use ``pre_call``.
|
||||
"""
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
if event_hooks is None:
|
||||
|
|
@ -1273,14 +1689,11 @@ class RealTimeStreaming:
|
|||
_check_data: Final = {**self.request_data, "transcript": transcript, "stream": True}
|
||||
_already_run: Final[set] = set()
|
||||
|
||||
for callback in litellm.callbacks:
|
||||
if not isinstance(callback, CustomGuardrail):
|
||||
continue
|
||||
if callback.use_native_lifecycle_hooks:
|
||||
continue
|
||||
if id(callback) in _already_run:
|
||||
continue
|
||||
if not any(callback.should_run_guardrail(data=_check_data, event_type=et) for et in _realtime_event_types):
|
||||
runner_guardrails: Final = (
|
||||
_realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks
|
||||
)
|
||||
for callback in runner_guardrails:
|
||||
if callback is None or id(callback) in _already_run:
|
||||
continue
|
||||
_already_run.add(id(callback))
|
||||
try:
|
||||
|
|
@ -1369,7 +1782,10 @@ class RealTimeStreaming:
|
|||
|
||||
async def _handle_provider_config_message(self, raw_response: str) -> None:
|
||||
"""Process a backend message when a provider_config is set (transformed path)."""
|
||||
returned_object: Final = self.provider_config.transform_realtime_response(
|
||||
provider_config: Final = self.provider_config
|
||||
if provider_config is None:
|
||||
raise RuntimeError("Provider response handling requires a provider configuration")
|
||||
returned_object: Final = provider_config.transform_realtime_response(
|
||||
raw_response,
|
||||
self.model,
|
||||
self.logging_obj,
|
||||
|
|
@ -1398,6 +1814,7 @@ class RealTimeStreaming:
|
|||
continue
|
||||
if isinstance(event, dict):
|
||||
self._detect_transcription_session_from_backend(event)
|
||||
self._capture_translation_output_audio(event)
|
||||
is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created"
|
||||
if is_session_created_event:
|
||||
if self._uses_deferred_backend_setup() and not self._backend_setup_complete:
|
||||
|
|
@ -1514,6 +1931,7 @@ class RealTimeStreaming:
|
|||
continue
|
||||
|
||||
if self.provider_config:
|
||||
self._backend_sent_non_error_frames = True
|
||||
try:
|
||||
await self._handle_provider_config_message(raw_response)
|
||||
except Exception as e:
|
||||
|
|
@ -1521,6 +1939,8 @@ class RealTimeStreaming:
|
|||
continue
|
||||
else:
|
||||
event = self._parse_backend_event(raw_response)
|
||||
if event is None or event.get("type") != "error":
|
||||
self._backend_sent_non_error_frames = True
|
||||
if event is None:
|
||||
await self.websocket.send_text(raw_response)
|
||||
continue
|
||||
|
|
@ -1535,6 +1955,7 @@ class RealTimeStreaming:
|
|||
verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
|
||||
continue
|
||||
|
||||
self._capture_translation_output_audio(event)
|
||||
if await self._handle_raw_backend_message(event, raw_response):
|
||||
continue
|
||||
|
||||
|
|
@ -1548,7 +1969,7 @@ class RealTimeStreaming:
|
|||
translated = self._translate_event_to_beta(event)
|
||||
if translated is None:
|
||||
continue
|
||||
await self.websocket.send_text(json.dumps(translated))
|
||||
await self.websocket.send_text(json.dumps(self._session_event_for_client(translated)))
|
||||
|
||||
async def backend_to_client_send_messages(self) -> BackendClose:
|
||||
import websockets
|
||||
|
|
@ -1573,8 +1994,8 @@ class RealTimeStreaming:
|
|||
|
||||
def _backend_refused_session(self, close: BackendClose) -> bool:
|
||||
return (
|
||||
close.code != 1000
|
||||
and not self._backend_sent_frames
|
||||
not self._backend_sent_non_error_frames
|
||||
and (close.code != 1000 or self._backend_sent_frames)
|
||||
and not self._holds_session_updates_until_first_content()
|
||||
)
|
||||
|
||||
|
|
@ -1617,7 +2038,7 @@ class RealTimeStreaming:
|
|||
_detect_beta_header = detect_beta_header
|
||||
|
||||
@staticmethod
|
||||
def _remap_beta_session_to_ga(session: dict) -> dict:
|
||||
def _remap_beta_session_to_ga(session: dict, all_beta_fields: bool = False, drop_temperature: bool = False) -> dict:
|
||||
"""
|
||||
Convert a beta-style session.update payload to the GA nested schema.
|
||||
|
||||
|
|
@ -1630,6 +2051,11 @@ class RealTimeStreaming:
|
|||
session.output_audio_format → session.audio.output.format (with type/rate)
|
||||
session.turn_detection → session.audio.input.turn_detection
|
||||
session.input_audio_transcription → session.audio.input.transcription
|
||||
Only with all_beta_fields:
|
||||
session.input_audio_noise_reduction → session.audio.input.noise_reduction
|
||||
session.speed → session.audio.output.speed
|
||||
session.max_response_output_tokens → session.max_output_tokens
|
||||
session.temperature (dropped when drop_temperature)
|
||||
─────────────────────────────────────────────────────────────────────
|
||||
Fields not in the mapping (instructions, tools, etc.) are passed through.
|
||||
GA clients that already use the nested shape are unaffected.
|
||||
|
|
@ -1681,6 +2107,19 @@ class RealTimeStreaming:
|
|||
if "input_audio_transcription" in session:
|
||||
inp["transcription"] = session.pop("input_audio_transcription")
|
||||
|
||||
if all_beta_fields and "input_audio_noise_reduction" in session:
|
||||
inp["noise_reduction"] = session.pop("input_audio_noise_reduction")
|
||||
|
||||
if all_beta_fields and "speed" in session:
|
||||
out["speed"] = session.pop("speed")
|
||||
|
||||
if all_beta_fields and "max_response_output_tokens" in session:
|
||||
max_tokens: Final = session.pop("max_response_output_tokens")
|
||||
session.setdefault("max_output_tokens", max_tokens)
|
||||
|
||||
if all_beta_fields and drop_temperature:
|
||||
session.pop("temperature", None)
|
||||
|
||||
if inp:
|
||||
audio["input"] = inp
|
||||
if out:
|
||||
|
|
@ -1699,6 +2138,53 @@ class RealTimeStreaming:
|
|||
|
||||
return session
|
||||
|
||||
@staticmethod
|
||||
def _remap_beta_response_to_ga(response: Mapping[str, object], drop_temperature: bool) -> dict[str, object]:
|
||||
moved: Final = frozenset(
|
||||
("modalities", "voice", "output_audio_format", "max_response_output_tokens")
|
||||
+ (("temperature",) if drop_temperature else ())
|
||||
)
|
||||
raw_format: Final = response.get("output_audio_format")
|
||||
output_audio: Final = {
|
||||
**({} if "voice" not in response else {"voice": response["voice"]}),
|
||||
**(
|
||||
{}
|
||||
if "output_audio_format" not in response
|
||||
else {
|
||||
"format": RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw_format, raw_format)
|
||||
if isinstance(raw_format, str)
|
||||
else raw_format
|
||||
}
|
||||
),
|
||||
}
|
||||
modalities: Final = response.get("modalities")
|
||||
modality_names: Final = (
|
||||
frozenset(m.lower() for m in modalities if isinstance(m, str))
|
||||
if isinstance(modalities, list)
|
||||
else frozenset()
|
||||
)
|
||||
output_modalities: Final = (
|
||||
["audio"] if "audio" in modality_names else ["text"] if "text" in modality_names else None
|
||||
)
|
||||
existing_audio: Final = as_str_mapping(response.get("audio")) or {}
|
||||
existing_output: Final = as_str_mapping(existing_audio.get("output")) or {}
|
||||
return {
|
||||
**{key: value for key, value in response.items() if key not in moved},
|
||||
**(
|
||||
{}
|
||||
if output_modalities is None or "output_modalities" in response
|
||||
else {"output_modalities": output_modalities}
|
||||
),
|
||||
**(
|
||||
{}
|
||||
if "max_response_output_tokens" not in response or "max_output_tokens" in response
|
||||
else {"max_output_tokens": response["max_response_output_tokens"]}
|
||||
),
|
||||
**(
|
||||
{} if not output_audio else {"audio": {**existing_audio, "output": {**existing_output, **output_audio}}}
|
||||
),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _translate_event_to_beta(event: dict) -> dict | None:
|
||||
"""Translate a single GA event dict to its beta equivalent.
|
||||
|
|
@ -1932,10 +2418,34 @@ class RealTimeStreaming:
|
|||
if msg_type == "session.update" and not self._backend_uses_beta_protocol:
|
||||
session = client_event.get("session", {})
|
||||
if isinstance(session, dict):
|
||||
session = self._remap_beta_session_to_ga(session)
|
||||
session = self._remap_beta_session_to_ga(
|
||||
session,
|
||||
all_beta_fields=self._remaps_all_beta_fields_to_ga,
|
||||
drop_temperature=self._drop_params,
|
||||
)
|
||||
if self._is_translation_session:
|
||||
session.pop("type", None)
|
||||
msg_obj["session"] = session
|
||||
message = json.dumps(msg_obj)
|
||||
|
||||
if (
|
||||
msg_type == "response.create"
|
||||
and self._remaps_all_beta_fields_to_ga
|
||||
and (beta_response := as_str_mapping(client_event.get("response"))) is not None
|
||||
):
|
||||
msg_obj["response"] = self._remap_beta_response_to_ga(
|
||||
beta_response, drop_temperature=self._drop_params
|
||||
)
|
||||
message = json.dumps(msg_obj)
|
||||
|
||||
if (
|
||||
msg_type == "conversation.item.create"
|
||||
and self._bridges_beta_client_to_ga
|
||||
and (beta_item := as_str_mapping(client_event.get("item"))) is not None
|
||||
):
|
||||
msg_obj["item"] = beta_item_to_ga(beta_item)
|
||||
message = json.dumps(msg_obj)
|
||||
|
||||
if msg_type == "session.update" and self._event_normalizer:
|
||||
session = client_event.get("session")
|
||||
if isinstance(session, dict):
|
||||
|
|
@ -2019,6 +2529,12 @@ class RealTimeStreaming:
|
|||
return ClientLoopExit.CLIENT_DISCONNECTED
|
||||
|
||||
async def bidirectional_forward(self) -> None:
|
||||
if self._is_translation_session and translation_guardrails_apply(self.request_data):
|
||||
await self.websocket.send_text(
|
||||
realtime_error_event(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, error_type="guardrail_error")
|
||||
)
|
||||
await self.websocket.close(code=1008, reason=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE)
|
||||
return
|
||||
forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages())
|
||||
client_task: Final = asyncio.create_task(self.client_ack_messages())
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
from collections.abc import Coroutine
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from openai import AsyncAzureOpenAI, AzureOpenAI
|
||||
from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream
|
||||
from openai.types.audio import TranscriptionStreamEvent
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import (
|
||||
acreate_sdk_transcription,
|
||||
create_sdk_transcription,
|
||||
sdk_compatible_transcription_request_data,
|
||||
)
|
||||
from litellm.types.utils import FileTypes
|
||||
from litellm.utils import (
|
||||
TranscriptionResponse,
|
||||
|
|
@ -38,13 +44,18 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
atranscription: bool = False,
|
||||
litellm_params: dict | None = None,
|
||||
custom_llm_provider: str = "azure",
|
||||
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
|
||||
) -> (
|
||||
TranscriptionResponse
|
||||
| Stream[TranscriptionStreamEvent]
|
||||
| Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
|
||||
):
|
||||
data: Final = {"model": model, "file": audio_file, **optional_params}
|
||||
sdk_data: Final = sdk_compatible_transcription_request_data(data)
|
||||
|
||||
if atranscription is True:
|
||||
return self.async_audio_transcriptions(
|
||||
audio_file=audio_file,
|
||||
data=data,
|
||||
data=sdk_data,
|
||||
model_response=model_response,
|
||||
timeout=timeout,
|
||||
api_key=api_key,
|
||||
|
|
@ -58,7 +69,7 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
)
|
||||
|
||||
azure_client: Final = self.get_azure_openai_client(
|
||||
api_version=api_version,
|
||||
api_version=None if self.is_azure_v1_api_version(api_version) else api_version,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
|
|
@ -84,10 +95,9 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
},
|
||||
)
|
||||
|
||||
response: Final = azure_client.audio.transcriptions.create(
|
||||
**data,
|
||||
timeout=timeout,
|
||||
)
|
||||
_, response = create_sdk_transcription(azure_client, sdk_data, timeout)
|
||||
if isinstance(response, Stream):
|
||||
return response
|
||||
|
||||
if isinstance(response, BaseModel):
|
||||
stringified_response = response.model_dump()
|
||||
|
|
@ -114,7 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
self,
|
||||
audio_file: FileTypes,
|
||||
model: str,
|
||||
data: dict,
|
||||
data: Mapping[str, object],
|
||||
model_response: TranscriptionResponse,
|
||||
timeout: float,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
|
|
@ -125,7 +135,7 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
max_retries=None,
|
||||
litellm_params: dict | None = None,
|
||||
custom_llm_provider: str = "azure",
|
||||
) -> TranscriptionResponse:
|
||||
) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]:
|
||||
response = None
|
||||
try:
|
||||
async_azure_client: Final = self.get_azure_openai_client(
|
||||
|
|
@ -155,12 +165,9 @@ class AzureAudioTranscription(AzureChatCompletion):
|
|||
},
|
||||
)
|
||||
|
||||
raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create(
|
||||
**data, timeout=timeout
|
||||
)
|
||||
|
||||
headers: Final = dict(raw_response.headers)
|
||||
response = raw_response.parse()
|
||||
headers, response = await acreate_sdk_transcription(async_azure_client, data, timeout)
|
||||
if isinstance(response, AsyncStream):
|
||||
return response
|
||||
|
||||
if isinstance(response, BaseModel):
|
||||
stringified_response = response.model_dump()
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
|
|
@ -48,17 +50,55 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe
|
|||
pass
|
||||
|
||||
|
||||
def azure_realtime_requires_ga(model: str) -> bool:
|
||||
try:
|
||||
azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure")
|
||||
except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly
|
||||
try:
|
||||
openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai")
|
||||
except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly
|
||||
return False
|
||||
openai_entry: Final = openai_model_info.get("provider_specific_entry")
|
||||
return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1
|
||||
azure_entry: Final = azure_model_info.get("provider_specific_entry")
|
||||
return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1
|
||||
|
||||
|
||||
def _is_ga_protocol(protocol: str | None) -> bool:
|
||||
return protocol is not None and protocol.upper() in ("GA", "V1")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AzureRealtimeProtocol:
|
||||
name: str
|
||||
bridges_beta_client_to_ga: bool = False
|
||||
|
||||
|
||||
def azure_realtime_protocol_for_client(
|
||||
configured_protocol: object,
|
||||
*,
|
||||
model: str,
|
||||
base_model: str | None = None,
|
||||
realtime_mode: str,
|
||||
query_params: RealtimeQueryParams | None,
|
||||
websocket: ScopedWebSocket,
|
||||
) -> str:
|
||||
) -> AzureRealtimeProtocol:
|
||||
serves_a_ga_only_session: Final = (
|
||||
realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription"
|
||||
)
|
||||
client_wants_beta: Final = client_sent_openai_beta_realtime_header(websocket)
|
||||
if isinstance(configured_protocol, str) and configured_protocol:
|
||||
return configured_protocol
|
||||
if (query_params or {}).get("intent") == "transcription":
|
||||
return "GA"
|
||||
return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA"
|
||||
return AzureRealtimeProtocol(
|
||||
name=configured_protocol,
|
||||
bridges_beta_client_to_ga=client_wants_beta
|
||||
and not serves_a_ga_only_session
|
||||
and _is_ga_protocol(configured_protocol),
|
||||
)
|
||||
if serves_a_ga_only_session:
|
||||
return AzureRealtimeProtocol(name="GA")
|
||||
if azure_realtime_requires_ga(base_model or model):
|
||||
return AzureRealtimeProtocol(name="GA", bridges_beta_client_to_ga=client_wants_beta)
|
||||
return AzureRealtimeProtocol(name="beta" if client_wants_beta else "GA")
|
||||
|
||||
|
||||
class _ProxyClientWebSocket(Protocol):
|
||||
|
|
@ -92,6 +132,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
api_version: str | None,
|
||||
realtime_protocol: str | None = None,
|
||||
query_params: RealtimeQueryParams | None = None,
|
||||
realtime_mode: str = "realtime",
|
||||
) -> str:
|
||||
"""
|
||||
Construct Azure realtime WebSocket URL.
|
||||
|
|
@ -104,6 +145,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
- "GA" or "v1": Uses /openai/v1/realtime (GA path)
|
||||
- "beta" or None: Uses /openai/realtime (beta path, default)
|
||||
query_params: Extra query params to forward (e.g. intent=transcription).
|
||||
realtime_mode: Realtime surface to target, "realtime" or "translation".
|
||||
|
||||
Returns:
|
||||
WebSocket URL string
|
||||
|
|
@ -117,24 +159,29 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
api_base = api_base.replace("https://", "wss://")
|
||||
|
||||
# Determine path based on realtime_protocol (case-insensitive)
|
||||
_is_ga: Final = realtime_protocol is not None and realtime_protocol.upper() in (
|
||||
"GA",
|
||||
"V1",
|
||||
)
|
||||
_is_ga: Final = _is_ga_protocol(realtime_protocol)
|
||||
intent: Final = (query_params or {}).get("intent")
|
||||
|
||||
if _is_ga:
|
||||
path = "/openai/v1/realtime"
|
||||
query_parts = []
|
||||
if intent != "transcription" and (query_params is None or "model" in query_params):
|
||||
query_parts.append(urlencode({"model": model}))
|
||||
else:
|
||||
# Default to beta path for backwards compatibility
|
||||
path = "/openai/realtime"
|
||||
query_parts = [urlencode({"api-version": api_version, "deployment": model})]
|
||||
path: Final = (
|
||||
"/openai/v1/realtime/translations"
|
||||
if realtime_mode == "translation"
|
||||
else "/openai/v1/realtime"
|
||||
if _is_ga
|
||||
else "/openai/realtime"
|
||||
)
|
||||
base_query_parts: Final = (
|
||||
(urlencode((("model", model),)),)
|
||||
if realtime_mode == "translation"
|
||||
else (
|
||||
(urlencode((("model", model),)),)
|
||||
if intent != "transcription" and (query_params is None or "model" in query_params)
|
||||
else ()
|
||||
)
|
||||
if _is_ga
|
||||
else (urlencode((("api-version", api_version), ("deployment", model))),)
|
||||
)
|
||||
|
||||
if intent:
|
||||
query_parts.append(urlencode({"intent": intent}))
|
||||
query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts
|
||||
|
||||
qs: Final = "&".join(query_parts)
|
||||
endpoint_url: Final = join_api_base(api_base, path)
|
||||
|
|
@ -165,13 +212,16 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
query_params: RealtimeQueryParams | None = None,
|
||||
user_api_key_dict: object | None = None,
|
||||
litellm_metadata: dict | None = None,
|
||||
realtime_mode: str = "realtime",
|
||||
bridges_beta_client_to_ga: bool = False,
|
||||
drop_params: bool = False,
|
||||
):
|
||||
import websockets
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required for Azure OpenAI calls")
|
||||
backend_uses_beta_protocol: Final = realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")
|
||||
backend_uses_beta_protocol: Final = not _is_ga_protocol(realtime_protocol)
|
||||
if api_version is None and backend_uses_beta_protocol:
|
||||
raise ValueError("api_version is required for Azure OpenAI calls")
|
||||
|
||||
|
|
@ -181,6 +231,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
api_version,
|
||||
realtime_protocol=realtime_protocol,
|
||||
query_params=query_params,
|
||||
realtime_mode=realtime_mode,
|
||||
)
|
||||
|
||||
auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token)
|
||||
|
|
@ -204,6 +255,9 @@ class AzureOpenAIRealtime(AzureChatCompletion):
|
|||
force_transcription_model=(
|
||||
model if (query_params or {}).get("intent") == "transcription" else None
|
||||
),
|
||||
translation_session=realtime_mode == "translation",
|
||||
bridges_beta_client_to_ga=bridges_beta_client_to_ga,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
await realtime_streaming.bidirectional_forward()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,20 @@
|
|||
"""Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls)."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.azure.common_utils import get_azure_ad_token
|
||||
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_CALLER_AUTH_HEADERS: Final = frozenset({"authorization", "api-key"})
|
||||
|
||||
|
||||
def _azure_resource_base(api_base: str) -> str:
|
||||
return api_base.rstrip("/").removesuffix("/openai/v1").removesuffix("/openai")
|
||||
|
||||
|
||||
def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
|
||||
|
|
@ -21,6 +30,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
|
|||
def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
|
||||
return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
|
||||
|
||||
def get_extra_headers(
|
||||
self,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
api_key: str,
|
||||
extra_headers: Mapping[str, object] | None,
|
||||
) -> Mapping[str, object] | None:
|
||||
if api_key or any(name.lower() in _CALLER_AUTH_HEADERS for name in extra_headers or {}):
|
||||
return extra_headers
|
||||
azure_ad_token: Final = get_azure_ad_token(litellm_params)
|
||||
if azure_ad_token is None:
|
||||
return extra_headers
|
||||
return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"}
|
||||
|
||||
def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
|
||||
|
||||
|
|
@ -30,20 +52,30 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
|
|||
model: str,
|
||||
api_key: str | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
validated_headers: Final = {
|
||||
**headers,
|
||||
"api-key": api_key or "",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
if api_key:
|
||||
validated_headers["api-key"] = api_key
|
||||
return validated_headers
|
||||
|
||||
def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
|
||||
|
||||
def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
base: Final = self.get_api_base(api_base)
|
||||
base: Final = _azure_resource_base(self.get_api_base(api_base))
|
||||
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
|
||||
return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}"
|
||||
|
||||
def get_translation_client_secret_url(
|
||||
self, api_base: str | None, model: str, api_version: str | None = None
|
||||
) -> str:
|
||||
return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/client_secrets")
|
||||
|
||||
def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/calls")
|
||||
|
||||
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {ephemeral_key}",
|
||||
|
|
|
|||
|
|
@ -1,9 +1,30 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Final,
|
||||
cast, # noqa: TID251 # SDK file handles have no runtime validation schema
|
||||
)
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
|
||||
from openai.types.audio import (
|
||||
Transcription,
|
||||
TranscriptionDiarized,
|
||||
TranscriptionStreamEvent,
|
||||
TranscriptionVerbose,
|
||||
)
|
||||
from openai.types.audio.transcription_create_params import (
|
||||
TranscriptionCreateParamsBase,
|
||||
TranscriptionCreateParamsStreaming,
|
||||
)
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -11,6 +32,31 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
|
||||
|
||||
_TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object])
|
||||
_TRANSCRIPTION_TEXT: Final = TypeAdapter(str)
|
||||
_TRANSCRIPTION_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_TRANSCRIPTION_TYPES_BY_FORMAT: Final = MappingProxyType(
|
||||
{"verbose_json": TranscriptionVerbose, "diarized_json": TranscriptionDiarized}
|
||||
)
|
||||
|
||||
|
||||
def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]:
|
||||
request_data: Final = _TRANSCRIPTION_REQUEST.validate_python(data)
|
||||
extension_keys: Final = ("keywords", "languages")
|
||||
extension_body: Final = {key: request_data[key] for key in extension_keys if key in request_data}
|
||||
if not extension_body:
|
||||
return request_data
|
||||
|
||||
existing_extra_body: Final = request_data.get("extra_body")
|
||||
return {
|
||||
**{key: value for key, value in request_data.items() if key not in extension_keys},
|
||||
"extra_body": {
|
||||
**(as_str_mapping(existing_extra_body) or {}),
|
||||
**extension_body,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
|
@ -20,6 +66,77 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
def sdk_compatible_transcription_response(response: object) -> BaseModel | str:
|
||||
if isinstance(response, BaseModel):
|
||||
return response
|
||||
return _TRANSCRIPTION_TEXT.validate_python(response, strict=True)
|
||||
|
||||
|
||||
def _content_type(http_response: httpx.Response) -> str:
|
||||
return next(iter(http_response.headers.get_list("content-type")), "")
|
||||
|
||||
|
||||
def _is_event_stream(http_response: httpx.Response) -> bool:
|
||||
return _content_type(http_response).startswith("text/event-stream")
|
||||
|
||||
|
||||
def _unstreamed_transcription_response(http_response: httpx.Response, response_format: object) -> BaseModel | str:
|
||||
if not _content_type(http_response).startswith("application/json"):
|
||||
return http_response.text
|
||||
transcription_type: Final = _TRANSCRIPTION_TYPES_BY_FORMAT.get(str(response_format), Transcription)
|
||||
return transcription_type.model_construct(None, **_TRANSCRIPTION_BODY.validate_json(http_response.content))
|
||||
|
||||
|
||||
def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase:
|
||||
return cast( # cast-ok: SDK file handles have no runtime validation schema
|
||||
TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"}
|
||||
)
|
||||
|
||||
|
||||
def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsStreaming:
|
||||
return cast( # cast-ok: SDK file handles have no runtime validation schema
|
||||
TranscriptionCreateParamsStreaming, data
|
||||
)
|
||||
|
||||
|
||||
def create_sdk_transcription(
|
||||
client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout
|
||||
) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]:
|
||||
if not transcription_stream_requested(data.get("stream")):
|
||||
raw_response: Final = client.audio.transcriptions.with_raw_response.create(
|
||||
**_non_streaming_params(data), timeout=timeout
|
||||
)
|
||||
return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse())
|
||||
raw_stream: Final = client.audio.transcriptions.with_raw_response.create(
|
||||
**_streaming_params({**data, "stream": True}), timeout=timeout
|
||||
)
|
||||
if _is_event_stream(raw_stream.http_response):
|
||||
return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent])
|
||||
raw_stream.http_response.read()
|
||||
return dict(raw_stream.headers), _unstreamed_transcription_response(
|
||||
raw_stream.http_response, data.get("response_format")
|
||||
)
|
||||
|
||||
|
||||
async def acreate_sdk_transcription(
|
||||
client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout
|
||||
) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]:
|
||||
if not transcription_stream_requested(data.get("stream")):
|
||||
raw_response: Final = await client.audio.transcriptions.with_raw_response.create(
|
||||
**_non_streaming_params(data), timeout=timeout
|
||||
)
|
||||
return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse())
|
||||
raw_stream: Final = await client.audio.transcriptions.with_raw_response.create(
|
||||
**_streaming_params({**data, "stream": True}), timeout=timeout
|
||||
)
|
||||
if _is_event_stream(raw_stream.http_response):
|
||||
return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent])
|
||||
await raw_stream.http_response.aread()
|
||||
return dict(raw_stream.headers), _unstreamed_transcription_response(
|
||||
raw_stream.http_response, data.get("response_format")
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioTranscriptionRequestData:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow:
|
|||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
class BaseRealtimeHTTPConfig(ABC):
|
||||
|
|
@ -50,6 +53,14 @@ class BaseRealtimeHTTPConfig(ABC):
|
|||
explicit api_key → litellm.api_key → env var → ""
|
||||
"""
|
||||
|
||||
def get_extra_headers(
|
||||
self,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
api_key: str,
|
||||
extra_headers: Mapping[str, object] | None,
|
||||
) -> Mapping[str, object] | None:
|
||||
return extra_headers
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# client_secrets endpoint #
|
||||
# ------------------------------------------------------------------ #
|
||||
|
|
@ -64,6 +75,12 @@ class BaseRealtimeHTTPConfig(ABC):
|
|||
base = base.removesuffix("/v1")
|
||||
return join_api_base(base, "v1/realtime/transcription_sessions")
|
||||
|
||||
def get_translation_client_secret_url(
|
||||
self, api_base: str | None, model: str, api_version: str | None = None
|
||||
) -> str:
|
||||
base: Final = (api_base or "").rstrip("/")
|
||||
return f"{base}/v1/realtime/translations/client_secrets"
|
||||
|
||||
@abstractmethod
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
@ -86,6 +103,10 @@ class BaseRealtimeHTTPConfig(ABC):
|
|||
"""Return the full URL for POST /realtime/calls (SDP exchange)."""
|
||||
return join_api_base(api_base or "", "v1/realtime/calls")
|
||||
|
||||
def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
base: Final = (api_base or "").rstrip("/")
|
||||
return f"{base}/v1/realtime/translations/calls"
|
||||
|
||||
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
|
||||
"""
|
||||
Build headers for the realtime_calls POST.
|
||||
|
|
|
|||
|
|
@ -7065,7 +7065,7 @@ class BaseLLMHTTPHandler:
|
|||
timeout: float | httpx.Timeout,
|
||||
provider_config: BaseRealtimeHTTPConfig | None = None,
|
||||
model: str | None = None,
|
||||
extra_headers: dict[str, object] | None = None,
|
||||
extra_headers: Mapping[str, object] | None = None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None = None,
|
||||
api_version: str | None = None,
|
||||
) -> httpx.Response:
|
||||
|
|
@ -7098,7 +7098,7 @@ class BaseLLMHTTPHandler:
|
|||
timeout: float | httpx.Timeout,
|
||||
provider_config: BaseRealtimeHTTPConfig | None = None,
|
||||
model: str | None = None,
|
||||
extra_headers: dict[str, object] | None = None,
|
||||
extra_headers: Mapping[str, object] | None = None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None = None,
|
||||
api_version: str | None = None,
|
||||
) -> httpx.Response:
|
||||
|
|
@ -7117,9 +7117,8 @@ class BaseLLMHTTPHandler:
|
|||
api_version=api_version,
|
||||
)
|
||||
|
||||
async def _async_realtime_session_post(
|
||||
async def async_realtime_translation_client_secret_handler(
|
||||
self,
|
||||
endpoint: Literal["client_secrets", "transcription_sessions"],
|
||||
api_base: str,
|
||||
api_key: str,
|
||||
request_data: dict[str, object],
|
||||
|
|
@ -7127,7 +7126,35 @@ class BaseLLMHTTPHandler:
|
|||
timeout: float | httpx.Timeout,
|
||||
provider_config: BaseRealtimeHTTPConfig | None = None,
|
||||
model: str | None = None,
|
||||
extra_headers: dict[str, object] | None = None,
|
||||
extra_headers: Mapping[str, object] | None = None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None = None,
|
||||
api_version: str | None = None,
|
||||
) -> httpx.Response:
|
||||
return await self._async_realtime_session_post(
|
||||
endpoint="translation_client_secrets",
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
request_data=request_data,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
provider_config=provider_config,
|
||||
model=model,
|
||||
extra_headers=extra_headers,
|
||||
client=client,
|
||||
api_version=api_version,
|
||||
)
|
||||
|
||||
async def _async_realtime_session_post(
|
||||
self,
|
||||
endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"],
|
||||
api_base: str,
|
||||
api_key: str,
|
||||
request_data: dict[str, object],
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
timeout: float | httpx.Timeout,
|
||||
provider_config: BaseRealtimeHTTPConfig | None = None,
|
||||
model: str | None = None,
|
||||
extra_headers: Mapping[str, object] | None = None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None = None,
|
||||
api_version: str | None = None,
|
||||
) -> httpx.Response:
|
||||
|
|
@ -7150,13 +7177,20 @@ class BaseLLMHTTPHandler:
|
|||
url = provider_config.get_transcription_session_url(
|
||||
api_base=api_base, model=model or "", api_version=api_version
|
||||
)
|
||||
elif endpoint == "translation_client_secrets":
|
||||
url = provider_config.get_translation_client_secret_url(
|
||||
api_base=api_base, model=model or "", api_version=api_version
|
||||
)
|
||||
else:
|
||||
url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version)
|
||||
headers: dict[str, object] = provider_config.validate_environment(
|
||||
headers={}, model=model or "", api_key=api_key
|
||||
)
|
||||
else:
|
||||
url = join_api_base(api_base, f"v1/realtime/{endpoint}")
|
||||
endpoint_path: Final = (
|
||||
"translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint
|
||||
)
|
||||
url = join_api_base(api_base, f"v1/realtime/{endpoint_path}")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
|
|
@ -7190,6 +7224,12 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler:
|
||||
if isinstance(client, AsyncHTTPHandler):
|
||||
return client
|
||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI)
|
||||
|
||||
async def async_realtime_calls_handler(
|
||||
self,
|
||||
api_base: str,
|
||||
|
|
@ -7199,10 +7239,11 @@ class BaseLLMHTTPHandler:
|
|||
timeout: float | httpx.Timeout,
|
||||
provider_config: BaseRealtimeHTTPConfig | None = None,
|
||||
model: str | None = None,
|
||||
session_config: dict[str, object] | None = None,
|
||||
extra_headers: dict[str, object] | None = None,
|
||||
session_config: Mapping[str, object] | None = None,
|
||||
extra_headers: Mapping[str, object] | None = None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None = None,
|
||||
api_version: str | None = None,
|
||||
translation: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Forward POST /v1/realtime/calls (SDP exchange) to upstream provider.
|
||||
|
|
@ -7214,18 +7255,30 @@ class BaseLLMHTTPHandler:
|
|||
- sdp: the SDP offer (text)
|
||||
- session: JSON string with {"type": "realtime", "model": "...", ...}
|
||||
"""
|
||||
if client is None or not isinstance(client, AsyncHTTPHandler):
|
||||
async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders.OPENAI,
|
||||
)
|
||||
else:
|
||||
async_httpx_client = client
|
||||
session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally
|
||||
**(session_config or {})
|
||||
}
|
||||
if "type" not in session_data:
|
||||
session_data["type"] = "translation" if translation else "realtime"
|
||||
if "model" not in session_data and model:
|
||||
session_data["model"] = model
|
||||
|
||||
sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body
|
||||
|
||||
async_httpx_client: Final = self._get_realtime_async_http_client(client)
|
||||
|
||||
if provider_config is not None:
|
||||
url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version)
|
||||
url = (
|
||||
provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version)
|
||||
if translation
|
||||
else provider_config.get_realtime_calls_url(
|
||||
api_base=api_base, model=model or "", api_version=api_version
|
||||
)
|
||||
)
|
||||
headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key)
|
||||
else:
|
||||
url = join_api_base(api_base, "v1/realtime/calls")
|
||||
path: Final = "translations/calls" if translation else "calls"
|
||||
url = join_api_base(api_base, f"v1/realtime/{path}")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {openai_ephemeral_key}",
|
||||
}
|
||||
|
|
@ -7233,14 +7286,8 @@ class BaseLLMHTTPHandler:
|
|||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
# Build multipart form data: sdp + session JSON
|
||||
session_data: Final = session_config or {}
|
||||
if "type" not in session_data:
|
||||
session_data["type"] = "realtime"
|
||||
if "model" not in session_data and model:
|
||||
session_data["model"] = model
|
||||
|
||||
sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body
|
||||
if translation:
|
||||
headers["Content-Type"] = "application/sdp"
|
||||
|
||||
files: Final = {
|
||||
"sdp": (None, sdp_text, "text/plain"),
|
||||
|
|
@ -7258,12 +7305,14 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
|
||||
try:
|
||||
return await async_httpx_client.post(
|
||||
url=url,
|
||||
headers=headers,
|
||||
files=files,
|
||||
timeout=timeout,
|
||||
)
|
||||
if translation:
|
||||
return await async_httpx_client.post(
|
||||
url=url,
|
||||
headers=headers,
|
||||
content=sdp_text,
|
||||
timeout=timeout,
|
||||
)
|
||||
return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout)
|
||||
except Exception as e:
|
||||
if provider_config is not None:
|
||||
raise self._handle_error(
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
|
||||
return ssl_config
|
||||
|
||||
def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str:
|
||||
def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str:
|
||||
"""
|
||||
Construct the backend websocket URL with the client's query parameters.
|
||||
|
||||
|
|
@ -158,7 +158,8 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
api_base = api_base.replace("http://", "ws://")
|
||||
url = URL(api_base)
|
||||
# Set the correct path
|
||||
url = url.copy_with(path="/v1/realtime")
|
||||
path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime"
|
||||
url = url.copy_with(path=path)
|
||||
upstream_params: Final = tuple(
|
||||
(key, value) for key, value in query_params.items() if not (drops_model and key == "model")
|
||||
)
|
||||
|
|
@ -197,6 +198,8 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
query_params: RealtimeQueryParams | None = None,
|
||||
user_api_key_dict: object | None = None,
|
||||
litellm_metadata: dict | None = None,
|
||||
realtime_mode: str = "realtime",
|
||||
drop_params: bool = False,
|
||||
**kwargs: object,
|
||||
):
|
||||
import websockets
|
||||
|
|
@ -210,13 +213,15 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
# Use all query params if provided, else fallback to just model
|
||||
if query_params is None:
|
||||
query_params = {"model": model}
|
||||
url: Final = self._construct_url(api_base, query_params)
|
||||
url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode)
|
||||
|
||||
try:
|
||||
# Get provider-specific SSL configuration
|
||||
ssl_config: Final = self._get_ssl_config(url)
|
||||
|
||||
openai_beta_realtime: Final = client_sent_openai_beta_realtime_header(websocket)
|
||||
openai_beta_realtime: Final = realtime_mode != "translation" and client_sent_openai_beta_realtime_header(
|
||||
websocket
|
||||
)
|
||||
if not openai_beta_realtime:
|
||||
verbose_logger.debug(
|
||||
"OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). "
|
||||
|
|
@ -253,6 +258,8 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
),
|
||||
event_normalizer=self._make_event_normalizer(),
|
||||
usage_meter=self._make_usage_meter(),
|
||||
translation_session=realtime_mode == "translation",
|
||||
drop_params=drop_params,
|
||||
)
|
||||
await realtime_streaming.bidirectional_forward()
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
|
|||
base = base.removesuffix("/v1")
|
||||
return join_api_base(base, "v1/realtime/transcription_sessions")
|
||||
|
||||
def get_translation_client_secret_url(
|
||||
self, api_base: str | None, model: str, api_version: str | None = None
|
||||
) -> str:
|
||||
base = self.get_api_base(api_base).rstrip("/")
|
||||
base = base.removesuffix("/v1")
|
||||
return f"{base}/v1/realtime/translations/client_secrets"
|
||||
|
||||
def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
|
||||
base = self.get_api_base(api_base).rstrip("/")
|
||||
base = base.removesuffix("/v1")
|
||||
return f"{base}/v1/realtime/translations/calls"
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
|
|||
self,
|
||||
model: str,
|
||||
audio_file: FileTypes,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
) -> AudioTranscriptionRequestData:
|
||||
"""
|
||||
Transform the audio transcription request
|
||||
|
|
@ -37,3 +37,14 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
|
|||
return AudioTranscriptionRequestData(
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]:
|
||||
return [
|
||||
"prompt",
|
||||
"response_format",
|
||||
"keywords",
|
||||
"languages",
|
||||
"stream",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from collections.abc import Mapping
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from typing import TYPE_CHECKING, Final, Optional, cast
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
|
||||
from openai.types.audio import TranscriptionStreamEvent
|
||||
from pydantic import BaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -12,6 +13,9 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.audio_transcription.transformation import (
|
||||
BaseAudioTranscriptionConfig,
|
||||
acreate_sdk_transcription,
|
||||
create_sdk_transcription,
|
||||
sdk_compatible_transcription_request_data,
|
||||
)
|
||||
from litellm.types.utils import FileTypes
|
||||
from litellm.utils import (
|
||||
|
|
@ -28,31 +32,18 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
|||
async def make_openai_audio_transcriptions_request(
|
||||
self,
|
||||
openai_aclient: AsyncOpenAI,
|
||||
data: dict,
|
||||
data: Mapping[str, object],
|
||||
timeout: float | httpx.Timeout,
|
||||
):
|
||||
try:
|
||||
raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout)
|
||||
headers: Final = dict(raw_response.headers)
|
||||
response: Final = raw_response.parse()
|
||||
|
||||
return headers, response
|
||||
except Exception as e:
|
||||
raise e
|
||||
return await acreate_sdk_transcription(openai_aclient, sdk_compatible_transcription_request_data(data), timeout)
|
||||
|
||||
def make_sync_openai_audio_transcriptions_request(
|
||||
self,
|
||||
openai_client: OpenAI,
|
||||
data: dict,
|
||||
data: Mapping[str, object],
|
||||
timeout: float | httpx.Timeout,
|
||||
):
|
||||
try:
|
||||
raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout)
|
||||
headers: Final = dict(raw_response.headers)
|
||||
response: Final = raw_response.parse()
|
||||
return headers, response
|
||||
except Exception as e:
|
||||
raise e
|
||||
return create_sdk_transcription(openai_client, sdk_compatible_transcription_request_data(data), timeout)
|
||||
|
||||
def audio_transcriptions(
|
||||
self,
|
||||
|
|
@ -70,7 +61,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
|||
atranscription: bool = False,
|
||||
provider_config: BaseAudioTranscriptionConfig | None = None,
|
||||
shared_session: Optional["ClientSession"] = None,
|
||||
) -> TranscriptionResponse:
|
||||
) -> (
|
||||
TranscriptionResponse
|
||||
| Stream[TranscriptionStreamEvent]
|
||||
| Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
|
||||
):
|
||||
"""
|
||||
Handle audio transcription request
|
||||
"""
|
||||
|
|
@ -128,6 +123,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
|||
)
|
||||
logging_obj.model_call_details["response_headers"] = headers
|
||||
|
||||
if isinstance(response, Stream):
|
||||
return response
|
||||
|
||||
if isinstance(response, BaseModel):
|
||||
stringified_response = response.model_dump()
|
||||
else:
|
||||
|
|
@ -153,7 +151,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
|||
async def async_audio_transcriptions(
|
||||
self,
|
||||
audio_file: FileTypes,
|
||||
data: dict,
|
||||
data: Mapping[str, object],
|
||||
model_response: TranscriptionResponse,
|
||||
timeout: float,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
|
|
@ -192,6 +190,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
|
|||
timeout=timeout,
|
||||
)
|
||||
logging_obj.model_call_details["response_headers"] = headers
|
||||
if isinstance(response, AsyncStream):
|
||||
return response
|
||||
if isinstance(response, BaseModel):
|
||||
stringified_response = response.model_dump()
|
||||
else:
|
||||
|
|
|
|||
119
litellm/main.py
119
litellm/main.py
|
|
@ -37,6 +37,8 @@ if TYPE_CHECKING:
|
|||
import dotenv
|
||||
import httpx
|
||||
import openai
|
||||
from openai import AsyncStream, Stream
|
||||
from openai.types.audio import TranscriptionStreamEvent
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never, overload
|
||||
|
||||
|
|
@ -998,7 +1000,7 @@ def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object])
|
|||
|
||||
|
||||
def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TranscriptionResponse:
|
||||
return TranscriptionResponse(**cached_response_dict)
|
||||
return TranscriptionResponse.model_validate(cached_response_dict)
|
||||
|
||||
|
||||
def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TextCompletionResponse:
|
||||
|
|
@ -8495,7 +8497,10 @@ async def amoderation(
|
|||
|
||||
|
||||
@client
|
||||
async def atranscription(*args, **kwargs) -> TranscriptionResponse:
|
||||
async def atranscription(
|
||||
*args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility
|
||||
**kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility
|
||||
) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]:
|
||||
"""
|
||||
Calls openai + azure whisper endpoints.
|
||||
|
||||
|
|
@ -8539,6 +8544,14 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
|
|||
else:
|
||||
# Call the synchronous function using run_in_executor
|
||||
response = await loop.run_in_executor(None, func_with_context)
|
||||
if isinstance(response, AsyncStream):
|
||||
if file is not None:
|
||||
calculated_duration = calculate_request_duration(file)
|
||||
if calculated_duration is not None:
|
||||
response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it
|
||||
calculated_duration
|
||||
)
|
||||
return response
|
||||
if not isinstance(response, TranscriptionResponse):
|
||||
raise ValueError(
|
||||
f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}"
|
||||
|
|
@ -8551,9 +8564,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
|
|||
if response is not None and not isinstance(response, Coroutine) and file is not None:
|
||||
existing_duration: Final = getattr(response, "duration", None)
|
||||
if existing_duration is None:
|
||||
calculated_duration: Final = calculate_request_duration(file)
|
||||
if calculated_duration is not None:
|
||||
response.hidden_params["audio_transcription_duration"] = calculated_duration
|
||||
sync_calculated_duration: Final = calculate_request_duration(file)
|
||||
if sync_calculated_duration is not None:
|
||||
response.set_audio_transcription_duration(sync_calculated_duration)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
|
|
@ -8567,16 +8580,55 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
|
|||
)
|
||||
|
||||
|
||||
def _validate_gpt_transcription_request(
|
||||
model: str,
|
||||
pricing_model: str,
|
||||
custom_llm_provider: str,
|
||||
response_format: str | None,
|
||||
drop_params: object,
|
||||
) -> str | None:
|
||||
model_cost_key: Final = next(
|
||||
(
|
||||
key
|
||||
for key in (f"{custom_llm_provider}/{pricing_model}", pricing_model)
|
||||
if (as_str_mapping(litellm.model_cost.get(key)) or {}).get("litellm_provider") == custom_llm_provider
|
||||
),
|
||||
None,
|
||||
)
|
||||
model_info: Final = (
|
||||
get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider)
|
||||
if model_cost_key is not None
|
||||
else None
|
||||
)
|
||||
provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None
|
||||
if (
|
||||
provider_specific_entry is None
|
||||
or provider_specific_entry.get("transcription_json_or_text_only") != 1
|
||||
or response_format in (None, "json", "text")
|
||||
):
|
||||
return response_format
|
||||
if litellm.drop_params is True or normalize_drop_params(drop_params) is True:
|
||||
return None
|
||||
raise litellm.UnsupportedParamsError(
|
||||
message=f"{model} only supports response_format 'json' or 'text'",
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
|
||||
@client
|
||||
def transcription(
|
||||
model: str,
|
||||
file: FileTypes,
|
||||
## OPTIONAL OPENAI PARAMS ##
|
||||
language: str | None = None,
|
||||
languages: Sequence[str] | None = None,
|
||||
keywords: Sequence[str] | None = None,
|
||||
prompt: str | None = None,
|
||||
response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None,
|
||||
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
|
||||
temperature: int | None = None, # openai defaults this to 0
|
||||
stream: bool | None = None,
|
||||
## LITELLM PARAMS ##
|
||||
user: str | None = None,
|
||||
timeout=600, # default to 10 minutes
|
||||
|
|
@ -8587,7 +8639,11 @@ def transcription(
|
|||
custom_llm_provider=None,
|
||||
base_url: str | None = None,
|
||||
**kwargs,
|
||||
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
|
||||
) -> (
|
||||
TranscriptionResponse
|
||||
| Stream[TranscriptionStreamEvent]
|
||||
| Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
|
||||
):
|
||||
"""
|
||||
Calls openai + azure whisper endpoints.
|
||||
|
||||
|
|
@ -8625,19 +8681,36 @@ def transcription(
|
|||
|
||||
api_key = dynamic_api_key if dynamic_api_key is not None else api_key
|
||||
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
configured_base_model: Final = litellm_params_dict.get("base_model")
|
||||
pricing_model: Final = (
|
||||
configured_base_model.removeprefix(f"{custom_llm_provider}/")
|
||||
if isinstance(configured_base_model, str)
|
||||
else model
|
||||
)
|
||||
|
||||
supported_response_format: Final = _validate_gpt_transcription_request(
|
||||
model=model,
|
||||
pricing_model=pricing_model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
response_format=response_format,
|
||||
drop_params=kwargs.get("drop_params"),
|
||||
)
|
||||
|
||||
optional_params: Final = get_optional_params_transcription(
|
||||
model=model,
|
||||
language=language,
|
||||
languages=languages,
|
||||
keywords=keywords,
|
||||
prompt=prompt,
|
||||
response_format=response_format,
|
||||
response_format=supported_response_format,
|
||||
timestamp_granularities=timestamp_granularities,
|
||||
temperature=temperature,
|
||||
stream=stream,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
**non_default_params,
|
||||
)
|
||||
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
|
||||
litellm_logging_obj.update_environment_variables(
|
||||
model=model,
|
||||
user=user,
|
||||
|
|
@ -8654,11 +8727,17 @@ def transcription(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]
|
||||
response: (
|
||||
TranscriptionResponse
|
||||
| Stream[TranscriptionStreamEvent]
|
||||
| AsyncStream[TranscriptionStreamEvent]
|
||||
| Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
|
||||
)
|
||||
|
||||
provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
|
||||
model=model,
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
base_model=pricing_model,
|
||||
)
|
||||
|
||||
uses_openai_transport: Final = custom_llm_provider in OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS and not (
|
||||
|
|
@ -8669,7 +8748,7 @@ def transcription(
|
|||
# azure configs
|
||||
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
|
||||
|
||||
api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
|
||||
azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
|
||||
|
||||
azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN")
|
||||
|
||||
|
|
@ -8688,7 +8767,7 @@ def transcription(
|
|||
logging_obj=litellm_logging_obj,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
api_version=api_version,
|
||||
api_version=azure_api_version,
|
||||
azure_ad_token=azure_ad_token,
|
||||
max_retries=max_retries,
|
||||
litellm_params=litellm_params_dict,
|
||||
|
|
@ -8825,15 +8904,17 @@ def transcription(
|
|||
|
||||
# Store duration in _hidden_params for cost calculation without
|
||||
# exposing it in the response body (see sync path comment above).
|
||||
if not isinstance(response, Coroutine):
|
||||
existing_duration: Final = getattr(response, "duration", None)
|
||||
if existing_duration is None:
|
||||
calculated_duration: Final = calculate_request_duration(file)
|
||||
if not isinstance(response, Coroutine) and (
|
||||
isinstance(response, (Stream, AsyncStream)) or getattr(response, "duration", None) is None
|
||||
):
|
||||
calculated_duration: Final = calculate_request_duration(file)
|
||||
if isinstance(response, (Stream, AsyncStream)):
|
||||
if calculated_duration is not None:
|
||||
response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
|
||||
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
|
||||
response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it
|
||||
calculated_duration
|
||||
)
|
||||
response_hidden_params["audio_transcription_duration"] = calculated_duration
|
||||
elif calculated_duration is not None:
|
||||
response.set_audio_transcription_duration(calculated_duration)
|
||||
|
||||
return response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,168 @@
|
|||
{
|
||||
"azure/gpt-realtime-2.1-2026-07-07": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
"input_cost_per_image_token": 5e-06,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_image_token_cost": 5e-07,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"azure/gpt-realtime-2.1-mini-2026-07-07": {
|
||||
"cache_creation_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"input_cost_per_audio_token": 1e-05,
|
||||
"input_cost_per_image_token": 8e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_image_token_cost": 8e-08,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"azure/gpt-realtime-translate-2026-05-06": {
|
||||
"input_cost_per_second": 0.0005666666666666667,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio",
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-realtime-translate-2026-05-07": {
|
||||
"input_cost_per_second": 0.0005666666666666667,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://ai.azure.com/catalog/models/gpt-realtime-translate",
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio",
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-realtime-whisper-2026-05-06": {
|
||||
"input_cost_per_second": 0.0002833333333333333,
|
||||
"litellm_provider": "azure",
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/transcription_sessions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096
|
||||
},
|
||||
"azure/gpt-realtime-whisper-2026-05-07": {
|
||||
"input_cost_per_second": 0.0002833333333333333,
|
||||
"litellm_provider": "azure",
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/transcription_sessions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096
|
||||
},
|
||||
"sample_spec": {
|
||||
"code_interpreter_cost_per_session": 0.0,
|
||||
"computer_use_input_cost_per_1k_tokens": 0.0,
|
||||
|
|
@ -6424,6 +6588,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6458,6 +6625,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6491,6 +6661,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6525,6 +6698,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6547,6 +6723,7 @@
|
|||
"azure/gpt-realtime-2.1": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_image_token_cost": 5e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-06-25",
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
|
|
@ -6556,6 +6733,9 @@
|
|||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
|
||||
|
|
@ -6575,12 +6755,15 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure/gpt-realtime-2.1-mini": {
|
||||
"cache_creation_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_image_token_cost": 8e-08,
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"deprecation_date": "2027-06-25",
|
||||
"input_cost_per_audio_token": 1e-05,
|
||||
|
|
@ -6590,6 +6773,9 @@
|
|||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
|
||||
|
|
@ -6609,6 +6795,8 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -6626,6 +6814,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6659,6 +6850,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6791,6 +6985,9 @@
|
|||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime",
|
||||
|
|
@ -29215,9 +29412,9 @@
|
|||
},
|
||||
"gemini-3-pro-image": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_flex": 1e-07,
|
||||
"cache_read_input_token_cost_priority": 3.6e-07,
|
||||
"deprecation_date": "2027-05-28",
|
||||
|
|
@ -39042,6 +39239,9 @@
|
|||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
|
|
@ -39097,6 +39297,8 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -39133,6 +39335,8 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -52544,9 +52748,9 @@
|
|||
},
|
||||
"vertex_ai/gemini-3-pro-image": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_flex": 1e-07,
|
||||
"cache_read_input_token_cost_priority": 3.6e-07,
|
||||
"deprecation_date": "2027-05-28",
|
||||
|
|
@ -63346,6 +63550,10 @@
|
|||
"input_cost_per_second": 7.5e-05,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"transcription_json_or_text_only": 1,
|
||||
"transcription_usage_duration": 1
|
||||
},
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions",
|
||||
|
|
@ -63358,7 +63566,8 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"gpt-live-transcribe": {
|
||||
"input_cost_per_second": 0.000283333333333,
|
||||
|
|
@ -63376,7 +63585,8 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"gpt-live-1": {
|
||||
"input_cost_per_second": 0.000833333333333,
|
||||
|
|
@ -63410,7 +63620,8 @@
|
|||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true
|
||||
"supports_audio_output": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"claude-mythos-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
|
|
@ -71245,6 +71456,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
|
|
@ -71302,7 +71516,11 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-transcribe": {
|
||||
"deprecation_date": "2028-02-01",
|
||||
|
|
@ -71312,7 +71530,8 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions",
|
||||
"/v1/realtime/transcription_sessions"
|
||||
"/v1/realtime/transcription_sessions",
|
||||
"/v1/realtime"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -71321,7 +71540,13 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1,
|
||||
"transcription_json_or_text_only": 1,
|
||||
"transcription_usage_duration": 1
|
||||
},
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-realtime-translate": {
|
||||
"deprecation_date": "2027-11-09",
|
||||
|
|
@ -71339,7 +71564,11 @@
|
|||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true
|
||||
"supports_audio_output": true,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"aihubmix/agnes-2.5-flash": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
|
|
|
|||
|
|
@ -504,10 +504,13 @@ def _lazy_lock(app: "FastAPI", module_path: str) -> asyncio.Lock:
|
|||
|
||||
|
||||
def _register_feature(app: "FastAPI", feat: LazyFeature, module: object, features: tuple[LazyFeature, ...]) -> None:
|
||||
before: Final = len(app.router.routes)
|
||||
routes_before: Final = frozenset(id(route) for route in app.router.routes)
|
||||
feat.register_fn(app, module)
|
||||
lazy_routes: Final[Mapping[str, tuple[BaseRoute, ...]]] = MappingProxyType(
|
||||
{**_lazy_routes(app), feat.module_path: tuple(app.router.routes[before:])}
|
||||
{
|
||||
**_lazy_routes(app),
|
||||
feat.module_path: tuple(route for route in app.router.routes if id(route) not in routes_before),
|
||||
}
|
||||
)
|
||||
app.state.lazy_routes = lazy_routes # rebind-ok: the app owns the record of which routes each feature added
|
||||
app.router.routes[:] = _route_table( # rebind-ok: the app owns its route table
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -332,6 +332,8 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/realtime?{model}",
|
||||
"/v1/realtime?{model}",
|
||||
"/openai/v1/realtime?{model}",
|
||||
"/realtime/translations",
|
||||
"/v1/realtime/translations",
|
||||
# realtime (GA WebRTC HTTP routes)
|
||||
"/realtime/client_secrets",
|
||||
"/v1/realtime/client_secrets",
|
||||
|
|
@ -342,6 +344,10 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/realtime/transcription_sessions",
|
||||
"/v1/realtime/transcription_sessions",
|
||||
"/openai/v1/realtime/transcription_sessions",
|
||||
"/realtime/translations/client_secrets",
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
"/realtime/translations/calls",
|
||||
"/v1/realtime/translations/calls",
|
||||
# responses API
|
||||
"/responses",
|
||||
"/v1/responses",
|
||||
|
|
|
|||
|
|
@ -11,7 +11,15 @@ from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failur
|
|||
from litellm.types.agents import AgentResponse
|
||||
from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext
|
||||
|
||||
_MANAGED_REALTIME_ROUTES: Final = frozenset(("/realtime", "/v1/realtime", "/openai/v1/realtime"))
|
||||
_MANAGED_REALTIME_ROUTES: Final = frozenset(
|
||||
(
|
||||
"/realtime",
|
||||
"/v1/realtime",
|
||||
"/openai/v1/realtime",
|
||||
"/realtime/translations",
|
||||
"/v1/realtime/translations",
|
||||
)
|
||||
)
|
||||
_MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions"))
|
||||
_MANAGED_MODEL_ROUTES: Final = frozenset(
|
||||
f"{prefix}/{operation}"
|
||||
|
|
@ -112,7 +120,11 @@ def managed_inference_request(
|
|||
if route in _MANAGED_LIVE_ROUTES:
|
||||
return dict(body)
|
||||
if route in _MANAGED_REALTIME_ROUTES:
|
||||
model: Final = query_model or body.get("model")
|
||||
from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model
|
||||
|
||||
model: Final = resolve_realtime_route_model(
|
||||
query_model if isinstance(query_model, str) else None, None, route.endswith("/realtime/translations")
|
||||
)
|
||||
if not isinstance(model, str) or not model:
|
||||
raise_identity_failure(
|
||||
AgentIdentityFailure(message="Managed inference requires an explicit or configured model")
|
||||
|
|
|
|||
|
|
@ -1971,6 +1971,7 @@ _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",)
|
|||
# can_key_call_model() validates the real target model.
|
||||
_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS: Final = (
|
||||
"/realtime/client_secrets",
|
||||
"/realtime/translations/client_secrets",
|
||||
"/realtime/calls",
|
||||
)
|
||||
_MODEL_ROUTING_ID_FIELDS: Final = (
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
|
|||
populate_request_with_path_params,
|
||||
read_raw_json_body,
|
||||
read_request_body,
|
||||
resolve_realtime_route_model,
|
||||
rewrite_request_model,
|
||||
safe_get_request_headers,
|
||||
safe_get_request_query_params,
|
||||
|
|
@ -1246,11 +1247,21 @@ def _apply_budget_limits_to_end_user_params(
|
|||
|
||||
|
||||
async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model"))
|
||||
return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model", ""))
|
||||
|
||||
|
||||
async def user_api_key_auth_realtime_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
route_path: Final[object] = websocket.scope.get("path")
|
||||
model: Final = resolve_realtime_route_model(
|
||||
model=websocket.query_params.get("model"),
|
||||
intent=websocket.query_params.get("intent"),
|
||||
is_translation=isinstance(route_path, str) and route_path.endswith("/realtime/translations"),
|
||||
)
|
||||
return await user_api_key_auth_websocket_for_model(websocket, model=model or "")
|
||||
|
||||
|
||||
async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
return await user_api_key_auth_websocket_for_model(websocket, model=None)
|
||||
return await user_api_key_auth_websocket_for_model(websocket, model="")
|
||||
|
||||
|
||||
def _has_required_baggage_tag_mapping() -> bool:
|
||||
|
|
@ -1266,7 +1277,7 @@ def _has_required_baggage_tag_mapping() -> bool:
|
|||
|
||||
|
||||
async def user_api_key_auth_websocket_for_model(
|
||||
websocket: WebSocket, model: str | None, *, close_on_auth_error: bool = True
|
||||
websocket: WebSocket, model: str, *, close_on_auth_error: bool = True
|
||||
) -> UserAPIKeyAuth:
|
||||
ws_scope: Final = websocket.scope or {}
|
||||
scope_headers: Final = list(ws_scope.get("headers") or [])
|
||||
|
|
|
|||
|
|
@ -170,7 +170,9 @@ ProxyRouteType: TypeAlias = Literal[
|
|||
"_arealtime",
|
||||
"_aresponses_websocket",
|
||||
"acreate_realtime_client_secret",
|
||||
"acreate_realtime_translation_client_secret",
|
||||
"arealtime_calls",
|
||||
"arealtime_translation_calls",
|
||||
"aget_responses",
|
||||
"adelete_responses",
|
||||
"acancel_responses",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,16 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
_JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def resolve_realtime_route_model(model: str | None, intent: str | None, is_translation: bool) -> str | None:
|
||||
if model is not None:
|
||||
return model
|
||||
if is_translation:
|
||||
return "gpt-realtime-translate"
|
||||
if intent == "transcription":
|
||||
return "gpt-realtime-whisper"
|
||||
return None
|
||||
|
||||
|
||||
def resolve_inference_model(
|
||||
body_model: object,
|
||||
settings: Mapping[str, object],
|
||||
|
|
@ -572,14 +582,16 @@ async def get_form_data(request: Request) -> dict[str, Any]:
|
|||
Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]`
|
||||
"""
|
||||
form: Final = await request.form()
|
||||
parsed_form_data: Final[dict[str, Any]] = {}
|
||||
for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat
|
||||
if key.endswith("[]"):
|
||||
clean_key = key[:-2]
|
||||
parsed_form_data.setdefault(clean_key, []).append(value)
|
||||
else:
|
||||
parsed_form_data[key] = value
|
||||
return parsed_form_data
|
||||
form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items())
|
||||
array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]"))
|
||||
normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items)
|
||||
normalized_keys: Final = tuple(dict.fromkeys(key for key, _ in normalized_items))
|
||||
return {
|
||||
key: [value for item_key, value in normalized_items if item_key == key]
|
||||
if key in array_keys
|
||||
else next(value for item_key, value in reversed(normalized_items) if item_key == key)
|
||||
for key in normalized_keys
|
||||
}
|
||||
|
||||
|
||||
async def convert_upload_files_to_file_data(
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from functools import lru_cache
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -10,7 +11,7 @@ def realtime_request_body(model: str | None) -> bytes:
|
|||
Generate the realtime websocket request body. Cached with LRU semantics to avoid repeated
|
||||
string formatting work while keeping memory usage bounded.
|
||||
"""
|
||||
return f'{{"model": "{model or ""}"}}'.encode()
|
||||
return json.dumps({"model": model}).encode()
|
||||
|
||||
|
||||
_realtime_request_body: Final = realtime_request_body
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import websockets
|
||||
import websockets.exceptions
|
||||
from openai.types.audio import TranscriptionStreamEvent
|
||||
from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config
|
||||
from pydantic.fields import FieldInfo, PydanticUndefined
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never
|
||||
|
|
@ -207,6 +208,7 @@ from litellm.types.utils import (
|
|||
TextCompletionResponse,
|
||||
TokenCountMessagesFormat,
|
||||
TokenCountResponse,
|
||||
TranscriptionResponse,
|
||||
echoed_cost_map_pricing_fields,
|
||||
is_server_derived_pricing_key,
|
||||
pricing_override_fields,
|
||||
|
|
@ -365,6 +367,12 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
|||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
TRANSCRIPTION_RAN_UNSTREAMED,
|
||||
TRANSCRIPTION_STREAM_DONE_FRAME,
|
||||
guarded_transcription_events,
|
||||
transcription_stream_requested,
|
||||
)
|
||||
from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
|
|
@ -456,6 +464,7 @@ from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # lega
|
|||
user_api_key_auth,
|
||||
user_api_key_auth_accepted_websocket_for_model,
|
||||
user_api_key_auth_live_websocket,
|
||||
user_api_key_auth_realtime_websocket,
|
||||
user_api_key_auth_websocket,
|
||||
)
|
||||
from litellm.proxy.batches_endpoints.endpoints import router as batches_router
|
||||
|
|
@ -515,6 +524,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10
|
|||
resolve_inference_model,
|
||||
safe_get_request_headers,
|
||||
)
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
resolve_realtime_route_model as _resolve_realtime_route_model,
|
||||
)
|
||||
from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket
|
||||
from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
|
||||
from litellm.proxy.common_utils.model_listing_utils import (
|
||||
|
|
@ -534,9 +546,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
|
|||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
attribute_of,
|
||||
error_status_code,
|
||||
headers_with_litellm_call_id,
|
||||
litellm_call_id_headers,
|
||||
openai_error_param,
|
||||
openai_error_type,
|
||||
proxy_exception_for,
|
||||
route_failure_proxy_exception,
|
||||
with_litellm_call_id,
|
||||
|
|
@ -10551,6 +10565,24 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob
|
|||
return _resolve
|
||||
|
||||
|
||||
def _transcription_sse_error_frame(client_error: Exception) -> str:
|
||||
status_code: Final = error_status_code(client_error, 500)
|
||||
error_returned: Final = json.dumps(
|
||||
{
|
||||
"error": sse_error_payload(client_error)[1]
|
||||
if isinstance(client_error, HTTPException)
|
||||
else ProxyException(
|
||||
message=getattr(client_error, "message", str(client_error)),
|
||||
type=openai_error_type(client_error, status_code),
|
||||
param=openai_error_param(client_error),
|
||||
code=status_code,
|
||||
).to_dict()
|
||||
},
|
||||
default=str,
|
||||
)
|
||||
return f"data: {error_returned}\n\n"
|
||||
|
||||
|
||||
async def async_data_generator(
|
||||
response,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
@ -13583,11 +13615,13 @@ def _form_fields_without_file(form_data: Mapping[str, object]) -> Mapping[str, o
|
|||
"/v1/audio/transcriptions",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["audio"],
|
||||
responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}},
|
||||
)
|
||||
@router.post(
|
||||
"/audio/transcriptions",
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["audio"],
|
||||
responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}},
|
||||
)
|
||||
async def audio_transcriptions(
|
||||
request: Request,
|
||||
|
|
@ -13675,23 +13709,23 @@ async def audio_transcriptions(
|
|||
cache_key: Final = hidden_params.get("cache_key", None) or ""
|
||||
api_base: Final = hidden_params.get("api_base", None) or ""
|
||||
response_cost: Final = hidden_params.get("response_cost", None) or ""
|
||||
response_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
|
||||
response_call_id: Final = hidden_params.get("litellm_call_id", None) or litellm_call_id
|
||||
additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {}
|
||||
stream_logging_obj: Final = getattr(response, "logging_obj", None)
|
||||
|
||||
fastapi_response.headers.update(
|
||||
ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
model_id=model_id,
|
||||
cache_key=cache_key,
|
||||
api_base=api_base,
|
||||
version=version,
|
||||
response_cost=response_cost,
|
||||
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
|
||||
call_id=response_call_id,
|
||||
request_data=data,
|
||||
hidden_params=hidden_params,
|
||||
**additional_headers,
|
||||
)
|
||||
custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
model_id=model_id,
|
||||
cache_key=cache_key,
|
||||
api_base=api_base,
|
||||
version=version,
|
||||
response_cost=response_cost,
|
||||
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
|
||||
call_id=response_call_id,
|
||||
request_data=data,
|
||||
hidden_params=hidden_params,
|
||||
litellm_logging_obj=stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None,
|
||||
**additional_headers,
|
||||
)
|
||||
|
||||
# Call response headers hook (matches base_process_llm_request behavior)
|
||||
|
|
@ -13701,8 +13735,52 @@ async def audio_transcriptions(
|
|||
response=response,
|
||||
request_headers=dict(request.headers),
|
||||
)
|
||||
if callback_headers:
|
||||
fastapi_response.headers.update(callback_headers)
|
||||
response_headers: Final = {**custom_headers, **(callback_headers or {})}
|
||||
|
||||
ran_unstreamed: Final = (
|
||||
isinstance(response, TranscriptionResponse)
|
||||
and response.hidden_params.get(TRANSCRIPTION_RAN_UNSTREAMED) is True
|
||||
)
|
||||
if transcription_stream_requested(form_data.get("stream")) and (
|
||||
ran_unstreamed or hasattr(response, "__aiter__")
|
||||
):
|
||||
|
||||
async def transcription_event_stream(
|
||||
stream: AsyncIterator[TranscriptionStreamEvent],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
try:
|
||||
async for event in stream:
|
||||
yield f"data: {event.model_dump_json(exclude_unset=True)}\n\n"
|
||||
yield TRANSCRIPTION_STREAM_DONE_FRAME
|
||||
except Exception as stream_error: # noqa: BLE001 # stream boundary: a mid-stream failure ends the SSE body as an error event
|
||||
replacement: Final = await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=stream_error,
|
||||
request_data=(
|
||||
{**data, "litellm_logging_obj": stream_logging_obj}
|
||||
if isinstance(stream_logging_obj, LiteLLMLoggingObj)
|
||||
else data
|
||||
),
|
||||
)
|
||||
yield _transcription_sse_error_frame(stream_error if replacement is None else replacement)
|
||||
yield TRANSCRIPTION_STREAM_DONE_FRAME
|
||||
finally:
|
||||
with anyio.CancelScope(shield=True):
|
||||
close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None)
|
||||
if callable(close):
|
||||
close_result: Final = close()
|
||||
if inspect.isawaitable(close_result):
|
||||
await close_result
|
||||
|
||||
return StreamingResponse(
|
||||
guarded_transcription_events(response)
|
||||
if isinstance(response, TranscriptionResponse)
|
||||
else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)),
|
||||
media_type="text/event-stream",
|
||||
headers=response_headers,
|
||||
)
|
||||
|
||||
fastapi_response.headers.update(response_headers)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
|
|
@ -13848,6 +13926,8 @@ async def _reject_realtime_session(
|
|||
@app.websocket("/openai/v1/realtime")
|
||||
@app.websocket("/v1/realtime")
|
||||
@app.websocket("/realtime")
|
||||
@app.websocket("/v1/realtime/translations")
|
||||
@app.websocket("/realtime/translations")
|
||||
async def realtime_websocket_endpoint(
|
||||
websocket: WebSocket,
|
||||
model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."),
|
||||
|
|
@ -13856,7 +13936,7 @@ async def realtime_websocket_endpoint(
|
|||
None,
|
||||
description="Comma-separated list of guardrail names to apply to this request.",
|
||||
),
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket),
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_realtime_websocket),
|
||||
):
|
||||
requested_protocols: Final = tuple(
|
||||
protocol.strip()
|
||||
|
|
@ -13864,15 +13944,13 @@ async def realtime_websocket_endpoint(
|
|||
if protocol.strip()
|
||||
)
|
||||
|
||||
route_model = model
|
||||
is_translation: Final = websocket.url.path.endswith("/realtime/translations")
|
||||
route_model: Final = _resolve_realtime_route_model(model, intent, is_translation)
|
||||
if route_model is None:
|
||||
if intent == "transcription":
|
||||
route_model = "gpt-realtime-whisper"
|
||||
else:
|
||||
await _reject_realtime_session(
|
||||
websocket, user_api_key_dict, code=1008, reason="model query parameter is required"
|
||||
)
|
||||
return
|
||||
await _reject_realtime_session(
|
||||
websocket, user_api_key_dict, code=1008, reason="model query parameter is required"
|
||||
)
|
||||
return
|
||||
assert route_model is not None
|
||||
try:
|
||||
await can_key_call_resolved_model(
|
||||
|
|
@ -13893,6 +13971,7 @@ async def realtime_websocket_endpoint(
|
|||
model=model,
|
||||
intent=intent,
|
||||
guardrails=guardrails,
|
||||
is_translation=is_translation,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -13906,13 +13985,20 @@ async def _route_realtime_websocket_session(
|
|||
guardrails: str | None,
|
||||
request_path: str | None = None,
|
||||
live_session_start: Mapping[str, object] | None = None,
|
||||
is_translation: bool = False,
|
||||
) -> None:
|
||||
# Only use explicit parameters, not all query params
|
||||
query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent)))
|
||||
query_model: Final = route_model if is_translation else model
|
||||
query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys
|
||||
RealtimeQueryParams,
|
||||
dict(_realtime_query_params_template(query_model, intent)),
|
||||
)
|
||||
|
||||
data: dict[str, object] = {
|
||||
"model": route_model,
|
||||
"websocket": websocket,
|
||||
"query_params": query_params, # Only explicit params
|
||||
"realtime_mode": "translation" if is_translation else "realtime",
|
||||
}
|
||||
# Pass guardrails into data so pre-call guardrail processing picks them up
|
||||
if guardrails:
|
||||
|
|
|
|||
|
|
@ -2,19 +2,24 @@
|
|||
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Generator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi import status as http_status
|
||||
from pydantic import ConfigDict, TypeAdapter, with_config
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
||||
decrypt_value_helper,
|
||||
encrypt_value_helper,
|
||||
|
|
@ -43,7 +48,26 @@ router: Final = APIRouter()
|
|||
_REALTIME_TOKEN_VERSION: Final = "realtime_v1"
|
||||
_DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview"
|
||||
_DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper"
|
||||
_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription")
|
||||
_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _invalid_request_body_answers_400() -> Generator[None, None, None]:
|
||||
try:
|
||||
yield
|
||||
except ValidationError as error:
|
||||
raise ProxyException(message=str(error), type="invalid_request_error", param=None, code=400) from error
|
||||
|
||||
|
||||
_TRANSLATION_SECRET_BODY: Final = {
|
||||
"requestBody": inline_request_body(
|
||||
RealtimeClientSecretRequest,
|
||||
{"model": "gpt-realtime-translate", "session": {"type": "translation", "model": "gpt-realtime-translate"}},
|
||||
)
|
||||
}
|
||||
_TRANSLATION_SDP_BODY: Final = {
|
||||
"requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}}
|
||||
}
|
||||
|
||||
|
||||
@with_config(ConfigDict(strict=True, extra="allow"))
|
||||
|
|
@ -140,19 +164,60 @@ def _set_transcription_model_on_session(
|
|||
}
|
||||
|
||||
|
||||
def nested_transcription_upstream_model(model: str, team_id: str | None, llm_router: "Router | None") -> str:
|
||||
deployment_model: Final = (
|
||||
llm_router.registry().resolve(model, kind="upstream", team_id=team_id) if llm_router is not None else None
|
||||
)
|
||||
if deployment_model is None:
|
||||
return model
|
||||
try:
|
||||
return litellm.get_llm_provider(model=deployment_model)[0]
|
||||
except litellm.BadRequestError:
|
||||
return deployment_model
|
||||
|
||||
|
||||
async def _authorize_and_bind_nested_transcription_models(
|
||||
session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list
|
||||
llm_router: "Router | None",
|
||||
) -> None:
|
||||
nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data))
|
||||
for nested_model in nested_models:
|
||||
await can_key_call_resolved_model(
|
||||
model=nested_model,
|
||||
valid_token=user_api_key_dict,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if nested_models:
|
||||
_set_transcription_model_on_session(
|
||||
session=session_data,
|
||||
model=nested_transcription_upstream_model(nested_models[0], user_api_key_dict.team_id, llm_router),
|
||||
)
|
||||
|
||||
|
||||
async def _prepare_client_secret_session(
|
||||
req: RealtimeClientSecretRequest,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_model_list: list | None,
|
||||
llm_router: "Router | None",
|
||||
forced_session_type: str | None = None,
|
||||
) -> tuple[str, dict | None, str]:
|
||||
session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None)
|
||||
session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None
|
||||
requested_session_type: Final = req.session.type if req.session else None
|
||||
if forced_session_type is None and requested_session_type == "translation":
|
||||
raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint")
|
||||
session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type)
|
||||
session_data: Final[dict | None] = (
|
||||
req.session.model_dump(exclude_unset=True) if req.session else ({} if session_type == "translation" else None)
|
||||
)
|
||||
if session_data is not None:
|
||||
session_data["type"] = session_type
|
||||
|
||||
session_model: Final = req.session.model if req.session else None
|
||||
model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL
|
||||
requested_model: Final = session_model or req.model
|
||||
default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL
|
||||
model: str = requested_model or default_model
|
||||
if session_type != "transcription":
|
||||
await can_key_call_resolved_model(
|
||||
model=model,
|
||||
|
|
@ -160,6 +225,15 @@ async def _prepare_client_secret_session(
|
|||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if session_data is not None and (requested_model or session_type == "translation"):
|
||||
session_data["model"] = model
|
||||
if session_type == "translation":
|
||||
await _authorize_and_bind_nested_transcription_models(
|
||||
session_data=session_data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
return model, session_data, session_type
|
||||
|
||||
transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {})
|
||||
|
|
@ -194,12 +268,13 @@ def _encode_realtime_token_payload(
|
|||
team_id: str | None,
|
||||
expires_at: int | None,
|
||||
session_type: str = "realtime",
|
||||
caller_auth: Mapping[str, object] | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Encode metadata with the upstream ephemeral key so /realtime/calls can
|
||||
route without requiring model as a query param.
|
||||
"""
|
||||
payload: Final[dict[str, str | int | None]] = {
|
||||
payload: Final[Mapping[str, object]] = {
|
||||
"v": _REALTIME_TOKEN_VERSION,
|
||||
"ephemeral_key": ephemeral_key,
|
||||
"model_id": model_id,
|
||||
|
|
@ -207,10 +282,24 @@ def _encode_realtime_token_payload(
|
|||
"team_id": team_id or "",
|
||||
"expires_at": expires_at,
|
||||
"session_type": session_type,
|
||||
**({} if caller_auth is None else {"caller_auth": caller_auth}),
|
||||
}
|
||||
return json.dumps(payload, separators=(",", ":"))
|
||||
|
||||
|
||||
def _caller_auth_for_token(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]:
|
||||
return user_api_key_dict.model_dump(mode="json", exclude_none=True, exclude={"api_key", "parent_otel_span"})
|
||||
|
||||
|
||||
def _caller_auth_from_token(
|
||||
decoded_payload: Mapping[str, object] | None, user_id: str | None, team_id: str | None
|
||||
) -> UserAPIKeyAuth:
|
||||
caller_auth: Final = None if decoded_payload is None else as_str_mapping(decoded_payload.get("caller_auth"))
|
||||
if caller_auth is None:
|
||||
return UserAPIKeyAuth(user_id=user_id, team_id=team_id)
|
||||
return UserAPIKeyAuth.model_validate({**caller_auth, "api_key": caller_auth.get("token")})
|
||||
|
||||
|
||||
def _header_value(headers: Mapping[str, str], name: str, default: str) -> str:
|
||||
return headers.get(name, default)
|
||||
|
||||
|
|
@ -249,11 +338,24 @@ def _decode_realtime_token_payload(
|
|||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["realtime"],
|
||||
)
|
||||
@router.post(
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
openapi_extra=_TRANSLATION_SECRET_BODY,
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["realtime"],
|
||||
)
|
||||
@router.post(
|
||||
"/realtime/translations/client_secrets",
|
||||
openapi_extra=_TRANSLATION_SECRET_BODY,
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
tags=["realtime"],
|
||||
)
|
||||
async def create_realtime_client_secret(
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
) -> RealtimeClientSecretResponse:
|
||||
"""Create a short-lived client token for a Realtime conversation, transcription, or translation session"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
add_litellm_data_to_request,
|
||||
general_settings,
|
||||
|
|
@ -269,13 +371,16 @@ async def create_realtime_client_secret(
|
|||
data: dict = {}
|
||||
try:
|
||||
body: Final = await read_request_body(request=request)
|
||||
req: Final = RealtimeClientSecretRequest(**body)
|
||||
with _invalid_request_body_answers_400():
|
||||
req: Final = RealtimeClientSecretRequest.model_validate(body)
|
||||
is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path
|
||||
|
||||
model, session_data, session_type = await _prepare_client_secret_session(
|
||||
req=req,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_model_list=llm_model_list,
|
||||
llm_router=llm_router,
|
||||
forced_session_type="translation" if is_translation_request else None,
|
||||
)
|
||||
|
||||
data = {"model": model}
|
||||
|
|
@ -299,17 +404,20 @@ async def create_realtime_client_secret(
|
|||
proxy_config=proxy_config,
|
||||
)
|
||||
|
||||
call_type: Final = (
|
||||
"acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret"
|
||||
)
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=data,
|
||||
call_type="acreate_realtime_client_secret",
|
||||
call_type=call_type,
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model)
|
||||
|
||||
llm_call: Final = await route_request(
|
||||
data=data,
|
||||
route_type="acreate_realtime_client_secret",
|
||||
route_type=call_type,
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
|
|
@ -366,6 +474,7 @@ async def create_realtime_client_secret(
|
|||
team_id=getattr(user_api_key_dict, "team_id", None),
|
||||
expires_at=expires_at if isinstance(expires_at, int) else None,
|
||||
session_type=session_type,
|
||||
caller_auth=_caller_auth_for_token(user_api_key_dict) if session_type == "translation" else None,
|
||||
)
|
||||
encrypted_token: Final[str] = encrypt_value_helper(token_payload)
|
||||
upstream_json["value"] = encrypted_token
|
||||
|
|
@ -392,10 +501,27 @@ async def create_realtime_client_secret(
|
|||
"/openai/v1/realtime/calls",
|
||||
tags=["realtime"],
|
||||
)
|
||||
@router.post(
|
||||
"/v1/realtime/translations/calls",
|
||||
status_code=http_status.HTTP_201_CREATED,
|
||||
response_class=PlainTextResponse,
|
||||
response_description="WebRTC SDP answer",
|
||||
openapi_extra=_TRANSLATION_SDP_BODY,
|
||||
tags=["realtime"],
|
||||
)
|
||||
@router.post(
|
||||
"/realtime/translations/calls",
|
||||
status_code=http_status.HTTP_201_CREATED,
|
||||
response_class=PlainTextResponse,
|
||||
response_description="WebRTC SDP answer",
|
||||
openapi_extra=_TRANSLATION_SDP_BODY,
|
||||
tags=["realtime"],
|
||||
)
|
||||
async def proxy_realtime_calls(
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
) -> Response:
|
||||
"""Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
add_litellm_data_to_request,
|
||||
general_settings,
|
||||
|
|
@ -417,6 +543,7 @@ async def proxy_realtime_calls(
|
|||
media_type="application/json",
|
||||
)
|
||||
|
||||
is_translation_request: Final = "/realtime/translations/calls" in request.url.path
|
||||
encrypted_token: Final = auth_header.removeprefix("Bearer ").strip()
|
||||
decrypted_token_value: Final = decrypt_value_helper(
|
||||
value=encrypted_token,
|
||||
|
|
@ -434,33 +561,39 @@ async def proxy_realtime_calls(
|
|||
if decoded_payload is not None:
|
||||
# Check token expiry
|
||||
expires_at: Final = decoded_payload.get("expires_at")
|
||||
if expires_at is not None and isinstance(expires_at, int):
|
||||
if time.time() > expires_at:
|
||||
return Response(
|
||||
content=json.dumps({"error": "Token has expired"}),
|
||||
status_code=http_status.HTTP_401_UNAUTHORIZED,
|
||||
media_type="application/json",
|
||||
)
|
||||
if isinstance(expires_at, int) and time.time() > expires_at:
|
||||
return Response(
|
||||
content=json.dumps({"error": "Token has expired"}),
|
||||
status_code=http_status.HTTP_401_UNAUTHORIZED,
|
||||
media_type="application/json",
|
||||
)
|
||||
|
||||
openai_ephemeral_key = decoded_payload.get("ephemeral_key", "")
|
||||
model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL
|
||||
user_id = decoded_payload.get("user_id") or None
|
||||
team_id = decoded_payload.get("team_id") or None
|
||||
session_type = _coerce_realtime_session_type(decoded_payload.get("session_type"))
|
||||
raw_session_type: Final = decoded_payload.get("session_type")
|
||||
session_type = _coerce_realtime_session_type(raw_session_type)
|
||||
if is_translation_request != (raw_session_type == "translation"):
|
||||
return Response(
|
||||
content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}),
|
||||
status_code=http_status.HTTP_401_UNAUTHORIZED,
|
||||
media_type="application/json",
|
||||
)
|
||||
else:
|
||||
# Backward compatibility: older tokens contained only encrypted upstream key.
|
||||
if is_translation_request:
|
||||
return Response(
|
||||
content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}),
|
||||
status_code=http_status.HTTP_401_UNAUTHORIZED,
|
||||
media_type="application/json",
|
||||
)
|
||||
openai_ephemeral_key = decrypted_token_value
|
||||
model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL)
|
||||
user_id = None
|
||||
team_id = None
|
||||
session_type = "realtime"
|
||||
|
||||
# Build a minimal UserAPIKeyAuth with user/team IDs from the token
|
||||
# so spend tracking and budget enforcement work correctly.
|
||||
minimal_auth: Final = UserAPIKeyAuth(
|
||||
user_id=user_id,
|
||||
team_id=team_id,
|
||||
)
|
||||
token_caller_auth: Final = _caller_auth_from_token(decoded_payload, user_id, team_id)
|
||||
|
||||
data: dict = {}
|
||||
try:
|
||||
|
|
@ -487,22 +620,23 @@ async def proxy_realtime_calls(
|
|||
data=data,
|
||||
request=request,
|
||||
general_settings=general_settings,
|
||||
user_api_key_dict=minimal_auth,
|
||||
user_api_key_dict=token_caller_auth,
|
||||
version=version,
|
||||
proxy_config=proxy_config,
|
||||
)
|
||||
|
||||
call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls"
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=minimal_auth,
|
||||
user_api_key_dict=token_caller_auth,
|
||||
data=data,
|
||||
call_type="arealtime_calls",
|
||||
call_type=call_type,
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model)
|
||||
|
||||
llm_call: Final = await route_request(
|
||||
data=data,
|
||||
route_type="arealtime_calls",
|
||||
route_type=call_type,
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
|
|
@ -510,7 +644,7 @@ async def proxy_realtime_calls(
|
|||
|
||||
except Exception as e:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=minimal_auth,
|
||||
user_api_key_dict=token_caller_auth,
|
||||
original_exception=e,
|
||||
request_data=data,
|
||||
)
|
||||
|
|
@ -581,7 +715,8 @@ async def create_realtime_transcription_session(
|
|||
data: dict = {}
|
||||
try:
|
||||
body: Final = await read_request_body(request=request)
|
||||
req: Final = RealtimeTranscriptionSessionRequest(**body)
|
||||
with _invalid_request_body_answers_400():
|
||||
req: Final = RealtimeTranscriptionSessionRequest.model_validate(body)
|
||||
|
||||
model: Final[str] = req.resolved_model() or "gpt-realtime-whisper"
|
||||
await can_key_call_resolved_model(
|
||||
|
|
@ -592,6 +727,7 @@ async def create_realtime_transcription_session(
|
|||
)
|
||||
|
||||
transcription_session: Final = {k: v for k, v in body.items() if k != "model"}
|
||||
_set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True)
|
||||
data = {"model": model, "transcription_session": transcription_session}
|
||||
|
||||
data = await add_litellm_data_to_request(
|
||||
|
|
|
|||
|
|
@ -621,7 +621,9 @@ RouteType = Literal[
|
|||
"alist_input_items",
|
||||
"_arealtime", # private function for realtime API
|
||||
"acreate_realtime_client_secret",
|
||||
"acreate_realtime_translation_client_secret",
|
||||
"arealtime_calls",
|
||||
"arealtime_translation_calls",
|
||||
"acreate_realtime_transcription_session",
|
||||
"_aresponses_websocket", # private function for responses WebSocket mode
|
||||
"aimage_edit",
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@
|
|||
|
||||
import asyncio
|
||||
import os
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict, assert_never
|
||||
|
||||
|
|
@ -14,10 +16,14 @@ import litellm
|
|||
from litellm.constants import (
|
||||
AZURE_OPENAI_AUDIO_PROVIDERS,
|
||||
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
|
||||
request_timeout,
|
||||
)
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.xai.common_utils import XAIModelInfo
|
||||
|
|
@ -36,18 +42,24 @@ from litellm.utils import (
|
|||
ModelInfoLookupMiss,
|
||||
ProviderConfigManager,
|
||||
aresolve_api_key_secret_reference,
|
||||
get_base_model_from_metadata,
|
||||
lookup_model_info_for_registration,
|
||||
)
|
||||
|
||||
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
|
||||
from ..litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from ..litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from ..litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from ..litellm_core_utils.realtime_streaming import client_websocket_scope
|
||||
from ..litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
|
||||
from ..litellm_core_utils.realtime_streaming import (
|
||||
TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE,
|
||||
client_websocket_scope,
|
||||
translation_guardrails_apply,
|
||||
)
|
||||
from ..llms.azure.common_utils import get_azure_ad_token
|
||||
from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client
|
||||
from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed
|
||||
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
|
||||
from ..llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, realtime_ssl_for_url
|
||||
from ..llms.openai.common_utils import is_openai_backed_api_base
|
||||
from ..llms.openai.live.handler import OpenAILiveSessions
|
||||
from ..llms.openai.realtime.handler import (
|
||||
|
|
@ -66,8 +78,6 @@ from ..utils import client as wrapper_client
|
|||
if TYPE_CHECKING:
|
||||
from fastapi import WebSocket
|
||||
|
||||
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
|
||||
|
||||
azure_realtime: Final = AzureOpenAIRealtime()
|
||||
openai_realtime: Final = OpenAIRealtime()
|
||||
openai_live_sessions: Final = OpenAILiveSessions()
|
||||
|
|
@ -77,7 +87,7 @@ vertex_llm_base: Final = VertexBase()
|
|||
base_llm_http_handler = BaseLLMHTTPHandler()
|
||||
_EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -90,26 +100,145 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) ->
|
|||
return MappingProxyType({**credential_values, **model_params})
|
||||
|
||||
|
||||
def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]:
|
||||
if "model" not in session:
|
||||
return session
|
||||
return {**session, "model": model_name}
|
||||
def _transcription_model_overrides(session: Mapping[str, object], model_name: str) -> Mapping[str, object]:
|
||||
flat_transcription: Final = as_str_mapping(session.get("input_audio_transcription"))
|
||||
audio: Final = as_str_mapping(session.get("audio")) or {}
|
||||
audio_input: Final = as_str_mapping(audio.get("input")) or {}
|
||||
nested_transcription: Final = as_str_mapping(audio_input.get("transcription"))
|
||||
flat_part: Final = (
|
||||
{} if flat_transcription is None else {"input_audio_transcription": {**flat_transcription, "model": model_name}}
|
||||
)
|
||||
nested_part: Final = (
|
||||
{}
|
||||
if nested_transcription is None and flat_transcription is not None
|
||||
else {
|
||||
"audio": {
|
||||
**audio,
|
||||
"input": {**audio_input, "transcription": {**(nested_transcription or {}), "model": model_name}},
|
||||
}
|
||||
}
|
||||
)
|
||||
return MappingProxyType({**flat_part, **nested_part})
|
||||
|
||||
|
||||
def _resolved_request_session(request_data: Mapping[str, object], model_name: str) -> Mapping[str, object] | None:
|
||||
session: Final = as_str_mapping(request_data.get("session"))
|
||||
if not isinstance(session, dict):
|
||||
return None
|
||||
return _with_resolved_session_model(session, model_name)
|
||||
def with_resolved_session_model(session: Mapping[str, object], model_name: str) -> dict[str, object]:
|
||||
top_level_model: Final = {} if "model" not in session else {"model": model_name}
|
||||
transcription_models: Final = (
|
||||
_transcription_model_overrides(session, model_name) if session.get("type") == "transcription" else {}
|
||||
)
|
||||
return {**session, **top_level_model, **transcription_models}
|
||||
|
||||
|
||||
def _build_litellm_metadata(kwargs: dict) -> dict[str, object]:
|
||||
"""Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider)."""
|
||||
metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})}
|
||||
guardrails: Final = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or []
|
||||
if guardrails:
|
||||
metadata["guardrails"] = guardrails
|
||||
return metadata
|
||||
def _without_top_level_nulls(fields: Mapping[str, object]) -> dict[str, object]:
|
||||
return {key: value for key, value in fields.items() if value is not None}
|
||||
|
||||
|
||||
_POLICY_PIPELINE_METADATA_KEYS: Final = ("_guardrail_pipelines", "_pipeline_managed_guardrails")
|
||||
|
||||
|
||||
def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]:
|
||||
request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {})
|
||||
internal_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("litellm_metadata") or {})
|
||||
policy_pipeline_state: Final = {
|
||||
key: request_metadata[key] for key in _POLICY_PIPELINE_METADATA_KEYS if key in request_metadata
|
||||
}
|
||||
guardrails: Final = request_metadata.get("guardrails") or kwargs.get("guardrails")
|
||||
return {**policy_pipeline_state, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})}
|
||||
|
||||
|
||||
_GUARDRAIL_OPT_OUT_METADATA_KEYS: Final = ("user_api_key_metadata", "user_api_key_team_metadata")
|
||||
|
||||
|
||||
def _translation_litellm_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {})
|
||||
guardrail_opt_outs: Final = {
|
||||
key: request_metadata[key] for key in _GUARDRAIL_OPT_OUT_METADATA_KEYS if key in request_metadata
|
||||
}
|
||||
return MappingProxyType({**guardrail_opt_outs, **_build_litellm_metadata(kwargs)})
|
||||
|
||||
|
||||
_HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value})
|
||||
_WEBSOCKET_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, *AZURE_OPENAI_AUDIO_PROVIDERS})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TranslationProviderUnsupported:
|
||||
provider: str
|
||||
supported_providers: frozenset[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TranslationGuardrailsUnsupported:
|
||||
pass
|
||||
|
||||
|
||||
_TranslationRejection = _TranslationProviderUnsupported | _TranslationGuardrailsUnsupported
|
||||
|
||||
|
||||
def _translation_rejection(
|
||||
custom_llm_provider: str, supported_providers: frozenset[str], litellm_metadata: Mapping[str, object]
|
||||
) -> _TranslationRejection | None:
|
||||
if custom_llm_provider not in supported_providers:
|
||||
return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers)
|
||||
if translation_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}):
|
||||
return _TranslationGuardrailsUnsupported()
|
||||
return None
|
||||
|
||||
|
||||
def _translation_rejection_message(rejection: _TranslationRejection) -> str:
|
||||
match rejection:
|
||||
case _TranslationProviderUnsupported(provider=provider, supported_providers=supported_providers):
|
||||
return (
|
||||
f"Realtime translation supports only the {', '.join(sorted(supported_providers))} providers, "
|
||||
f"got {provider}"
|
||||
)
|
||||
case _TranslationGuardrailsUnsupported():
|
||||
return TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE
|
||||
case _:
|
||||
assert_never(rejection)
|
||||
|
||||
|
||||
def _translation_rejection_error_type(rejection: _TranslationRejection) -> str:
|
||||
match rejection:
|
||||
case _TranslationProviderUnsupported():
|
||||
return "invalid_request_error"
|
||||
case _TranslationGuardrailsUnsupported():
|
||||
return "guardrail_error"
|
||||
case _:
|
||||
assert_never(rejection)
|
||||
|
||||
|
||||
def _translation_rejection_error(
|
||||
rejection: _TranslationRejection, model: str, custom_llm_provider: str
|
||||
) -> BadRequestError:
|
||||
return BadRequestError(
|
||||
message=_translation_rejection_message(rejection), model=model, llm_provider=custom_llm_provider
|
||||
)
|
||||
|
||||
|
||||
def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None:
|
||||
rejection: Final = _translation_rejection(
|
||||
custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, _translation_litellm_metadata(kwargs)
|
||||
)
|
||||
if rejection is None:
|
||||
return
|
||||
raise _translation_rejection_error(rejection, model, custom_llm_provider)
|
||||
|
||||
|
||||
async def _refuse_translation_session(
|
||||
websocket: "WebSocket",
|
||||
rejection: _TranslationRejection,
|
||||
logging_obj: LiteLLMLogging,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
) -> None:
|
||||
message: Final = _translation_rejection_message(rejection)
|
||||
await websocket.send_text(realtime_error_event(message, error_type=_translation_rejection_error_type(rejection)))
|
||||
await websocket.close(code=1008, reason=websocket_close_reason(message, fallback="Translation session refused"))
|
||||
error: Final = _translation_rejection_error(rejection, model, custom_llm_provider)
|
||||
await logging_obj.dispatch_failure_handlers(
|
||||
error, "".join(traceback.format_exception(error)), prefer_async_handlers=True
|
||||
)
|
||||
|
||||
|
||||
def _get_realtime_http_provider_config(
|
||||
|
|
@ -125,10 +254,6 @@ def _get_realtime_http_provider_config(
|
|||
Uses ProviderConfigManager so each provider keeps its credential-resolution
|
||||
and URL-construction logic in its own transformation class.
|
||||
"""
|
||||
from litellm.llms.base_llm.realtime.http_transformation import (
|
||||
BaseRealtimeHTTPConfig,
|
||||
)
|
||||
|
||||
provider_config: BaseRealtimeHTTPConfig | None = None
|
||||
if custom_llm_provider in LlmProviders._member_map_.values():
|
||||
provider_config = ProviderConfigManager.get_provider_realtime_http_config(
|
||||
|
|
@ -163,6 +288,32 @@ async def _realtime_http_bearer_token(
|
|||
return bearer_token or ""
|
||||
|
||||
|
||||
async def _get_realtime_http_extra_headers(
|
||||
provider_config: BaseRealtimeHTTPConfig | None,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
resolved_api_key: str,
|
||||
extra_headers: Mapping[str, object] | None,
|
||||
) -> Mapping[str, object] | None:
|
||||
if provider_config is None:
|
||||
return extra_headers
|
||||
return await asyncio.to_thread(
|
||||
provider_config.get_extra_headers,
|
||||
litellm_params=litellm_params,
|
||||
api_key=resolved_api_key,
|
||||
extra_headers=extra_headers,
|
||||
)
|
||||
|
||||
|
||||
def _client_secret_session_body(req: RealtimeClientSecretRequest, model_name: str) -> Mapping[str, object] | None:
|
||||
if req.session is not None:
|
||||
return with_resolved_session_model(
|
||||
_without_top_level_nulls(req.session.model_dump(exclude_unset=True)), model_name
|
||||
)
|
||||
if req.model is not None:
|
||||
return {"type": "realtime", "model": model_name}
|
||||
return None
|
||||
|
||||
|
||||
@wrapper_client
|
||||
async def acreate_realtime_client_secret(
|
||||
model: str | None = None,
|
||||
|
|
@ -176,31 +327,42 @@ async def acreate_realtime_client_secret(
|
|||
session=RealtimeSessionConfig.model_validate(session) if session else None,
|
||||
expires_after=RealtimeExpiresAfter.model_validate(expires_after) if expires_after else None,
|
||||
)
|
||||
model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview"
|
||||
transcription_model: Final = (
|
||||
req.session.audio.input.transcription.model
|
||||
if req.session is not None
|
||||
and req.session.type == "transcription"
|
||||
and req.session.audio is not None
|
||||
and req.session.audio.input is not None
|
||||
and req.session.audio.input.transcription is not None
|
||||
else None
|
||||
)
|
||||
requested_model_name: Final = (
|
||||
transcription_model
|
||||
or (req.session.model if req.session is not None else None)
|
||||
or req.model
|
||||
or "gpt-4o-realtime-preview"
|
||||
)
|
||||
litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj")
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
||||
(
|
||||
model_name,
|
||||
custom_llm_provider,
|
||||
dynamic_api_key,
|
||||
dynamic_api_base,
|
||||
) = get_llm_provider(
|
||||
model=model_name,
|
||||
model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider(
|
||||
model=requested_model_name,
|
||||
custom_llm_provider=litellm_params.custom_llm_provider,
|
||||
api_base=litellm_params.api_base,
|
||||
api_key=litellm_params.api_key,
|
||||
)
|
||||
(
|
||||
provider_config,
|
||||
resolved_api_base,
|
||||
resolved_api_key,
|
||||
) = _get_realtime_http_provider_config(
|
||||
provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
dynamic_api_base=dynamic_api_base,
|
||||
dynamic_api_key=dynamic_api_key,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
resolved_extra_headers: Final = await _get_realtime_http_extra_headers(
|
||||
provider_config=provider_config,
|
||||
litellm_params=litellm_params,
|
||||
resolved_api_key=resolved_api_key,
|
||||
extra_headers=kwargs.get("extra_headers"),
|
||||
)
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model_name,
|
||||
|
|
@ -208,26 +370,124 @@ async def acreate_realtime_client_secret(
|
|||
litellm_params={"api_base": resolved_api_base},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
request_data: Final = req.model_dump(exclude_none=True, exclude={"model"})
|
||||
resolved_session: Final = _resolved_request_session(request_data, model_name)
|
||||
if resolved_session is not None:
|
||||
request_data["session"] = resolved_session
|
||||
session_body: Final = _client_secret_session_body(req, model_name)
|
||||
return await base_llm_http_handler.async_realtime_client_secret_handler(
|
||||
api_base=resolved_api_base,
|
||||
api_key=await _realtime_http_bearer_token(
|
||||
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
|
||||
),
|
||||
request_data=request_data,
|
||||
request_data={
|
||||
**req.model_dump(exclude_none=True, exclude={"model", "session"}),
|
||||
**({} if session_body is None else {"session": session_body}),
|
||||
},
|
||||
logging_obj=litellm_logging_obj,
|
||||
timeout=timeout or request_timeout,
|
||||
provider_config=provider_config,
|
||||
model=model_name,
|
||||
extra_headers=kwargs.get("extra_headers"),
|
||||
extra_headers=resolved_extra_headers,
|
||||
client=kwargs.get("client"),
|
||||
api_version=litellm_params.api_version,
|
||||
)
|
||||
|
||||
|
||||
@wrapper_client
|
||||
async def acreate_realtime_translation_client_secret(
|
||||
model: str | None = None,
|
||||
session: Mapping[str, Any] | None = None,
|
||||
expires_after: Mapping[str, Any] | None = None,
|
||||
timeout: float | None = None,
|
||||
**kwargs: object, # kwargs-ok: public API preserves provider-specific request options
|
||||
) -> httpx.Response:
|
||||
requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate"
|
||||
session_config: Final = RealtimeSessionConfig.model_validate(
|
||||
{
|
||||
**(session or {}),
|
||||
"type": "translation",
|
||||
"model": requested_model_name,
|
||||
}
|
||||
)
|
||||
req: Final = RealtimeClientSecretRequest(
|
||||
model=requested_model_name,
|
||||
session=session_config,
|
||||
expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None,
|
||||
)
|
||||
litellm_logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if not isinstance(litellm_logging_obj, LiteLLMLogging):
|
||||
raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance")
|
||||
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)
|
||||
|
||||
model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider(
|
||||
model=requested_model_name,
|
||||
custom_llm_provider=litellm_params.custom_llm_provider,
|
||||
api_base=litellm_params.api_base,
|
||||
api_key=litellm_params.api_key,
|
||||
)
|
||||
_raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs)
|
||||
provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
dynamic_api_base=dynamic_api_base,
|
||||
dynamic_api_key=dynamic_api_key,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
resolved_extra_headers: Final = await _get_realtime_http_extra_headers(
|
||||
provider_config=provider_config,
|
||||
litellm_params=litellm_params,
|
||||
resolved_api_key=resolved_api_key,
|
||||
extra_headers=as_str_mapping(kwargs.get("extra_headers")),
|
||||
)
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model_name,
|
||||
optional_params={
|
||||
"expires_after": expires_after,
|
||||
"session": session,
|
||||
},
|
||||
litellm_params={"api_base": resolved_api_base},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
translation_session: Final = with_resolved_session_model(
|
||||
_without_top_level_nulls(session_config.model_dump(exclude_unset=True)), model_name
|
||||
)
|
||||
return await base_llm_http_handler.async_realtime_translation_client_secret_handler(
|
||||
api_base=resolved_api_base,
|
||||
api_key=await _realtime_http_bearer_token(
|
||||
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
|
||||
),
|
||||
request_data={
|
||||
**req.model_dump(exclude_none=True, exclude={"model", "session"}),
|
||||
"session": {key: value for key, value in translation_session.items() if key != "type"},
|
||||
},
|
||||
logging_obj=litellm_logging_obj,
|
||||
timeout=timeout or request_timeout,
|
||||
provider_config=provider_config,
|
||||
model=model_name,
|
||||
extra_headers=resolved_extra_headers,
|
||||
client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None,
|
||||
api_version=litellm_params.api_version,
|
||||
)
|
||||
|
||||
|
||||
def _with_upstream_transcription_model(request_data: Mapping[str, object], model_name: str) -> dict[str, object]:
|
||||
flat: Final = as_str_mapping(request_data.get("input_audio_transcription"))
|
||||
audio: Final = as_str_mapping(request_data.get("audio")) or {}
|
||||
audio_input: Final = as_str_mapping(audio.get("input")) or {}
|
||||
nested: Final = as_str_mapping(audio_input.get("transcription"))
|
||||
return {
|
||||
**request_data,
|
||||
**({} if flat is None else {"input_audio_transcription": {**flat, "model": model_name}}),
|
||||
**(
|
||||
{}
|
||||
if nested is None
|
||||
else {"audio": {**audio, "input": {**audio_input, "transcription": {**nested, "model": model_name}}}}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: str) -> str | None:
|
||||
base_model: Final = get_base_model_from_metadata(model_call_details={"litellm_params": kwargs})
|
||||
return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/")
|
||||
|
||||
|
||||
@wrapper_client
|
||||
async def acreate_realtime_transcription_session(
|
||||
model: str | None = None,
|
||||
|
|
@ -273,6 +533,12 @@ async def acreate_realtime_transcription_session(
|
|||
dynamic_api_key=dynamic_api_key,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
resolved_extra_headers: Final = await _get_realtime_http_extra_headers(
|
||||
provider_config=provider_config,
|
||||
litellm_params=litellm_params,
|
||||
resolved_api_key=resolved_api_key,
|
||||
extra_headers=kwargs.get("extra_headers"),
|
||||
)
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model_name,
|
||||
|
|
@ -280,13 +546,11 @@ async def acreate_realtime_transcription_session(
|
|||
litellm_params={"api_base": resolved_api_base},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
request_data: Final = req.model_dump(exclude_none=True, exclude={"model"})
|
||||
# Ensure the upstream body's input_audio_transcription.model matches the
|
||||
# authorized routing model. This prevents a caller from supplying an allowed
|
||||
# top-level model for auth while sneaking a different model into the nested
|
||||
# transcription config that gets forwarded to the provider.
|
||||
if isinstance(request_data.get("input_audio_transcription"), dict):
|
||||
request_data["input_audio_transcription"]["model"] = model_name
|
||||
# The upstream transcription model must match the authorized routing model, so a
|
||||
# caller cannot pass auth with an allowed top-level model and forward a different one.
|
||||
request_data: Final = _with_upstream_transcription_model(
|
||||
_without_top_level_nulls(req.model_dump(exclude_unset=True, exclude={"model"})), model_name
|
||||
)
|
||||
return await base_llm_http_handler.async_realtime_transcription_session_handler(
|
||||
api_base=resolved_api_base,
|
||||
api_key=await _realtime_http_bearer_token(
|
||||
|
|
@ -297,7 +561,7 @@ async def acreate_realtime_transcription_session(
|
|||
timeout=timeout or request_timeout,
|
||||
provider_config=provider_config,
|
||||
model=model_name,
|
||||
extra_headers=kwargs.get("extra_headers"),
|
||||
extra_headers=resolved_extra_headers,
|
||||
client=kwargs.get("client"),
|
||||
api_version=litellm_params.api_version,
|
||||
)
|
||||
|
|
@ -334,7 +598,7 @@ async def arealtime_calls(
|
|||
litellm_params=litellm_params,
|
||||
)
|
||||
if session is not None:
|
||||
session = _with_resolved_session_model(session, model_name)
|
||||
session = with_resolved_session_model(session, model_name)
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model_name,
|
||||
|
|
@ -357,6 +621,68 @@ async def arealtime_calls(
|
|||
)
|
||||
|
||||
|
||||
@wrapper_client
|
||||
async def arealtime_translation_calls(
|
||||
openai_ephemeral_key: str,
|
||||
sdp_body: bytes,
|
||||
model: str | None = None,
|
||||
session: Mapping[str, Any] | None = None,
|
||||
timeout: float | None = None,
|
||||
**kwargs: object, # kwargs-ok: public API preserves provider-specific request options
|
||||
) -> httpx.Response:
|
||||
requested_model_name: Final = model or "gpt-realtime-translate"
|
||||
litellm_logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if not isinstance(litellm_logging_obj, LiteLLMLogging):
|
||||
raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance")
|
||||
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)
|
||||
|
||||
model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider(
|
||||
model=requested_model_name,
|
||||
custom_llm_provider=litellm_params.custom_llm_provider,
|
||||
api_base=litellm_params.api_base,
|
||||
api_key=litellm_params.api_key,
|
||||
)
|
||||
_raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs)
|
||||
provider_config, resolved_api_base, _ = _get_realtime_http_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
dynamic_api_base=dynamic_api_base,
|
||||
dynamic_api_key=dynamic_api_key,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
session_config: Final = with_resolved_session_model(
|
||||
{
|
||||
**(session or {}),
|
||||
"type": "translation",
|
||||
"model": model_name,
|
||||
},
|
||||
model_name,
|
||||
)
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
model=model_name,
|
||||
optional_params={
|
||||
"realtime_translation_calls": True,
|
||||
"session": session_config,
|
||||
},
|
||||
litellm_params={"api_base": resolved_api_base},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
return await base_llm_http_handler.async_realtime_calls_handler(
|
||||
api_base=resolved_api_base,
|
||||
openai_ephemeral_key=openai_ephemeral_key,
|
||||
sdp_body=sdp_body,
|
||||
logging_obj=litellm_logging_obj,
|
||||
timeout=timeout or request_timeout,
|
||||
provider_config=provider_config,
|
||||
model=model_name,
|
||||
session_config=session_config,
|
||||
extra_headers=as_str_mapping(kwargs.get("extra_headers")),
|
||||
client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None,
|
||||
api_version=litellm_params.api_version,
|
||||
translation=True,
|
||||
)
|
||||
|
||||
|
||||
async def vertex_access_token_resolver(
|
||||
credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
project_id: str | None,
|
||||
|
|
@ -486,6 +812,7 @@ async def _arealtime(
|
|||
timeout: float | None = None,
|
||||
query_params: RealtimeQueryParams | None = None,
|
||||
live_session_start: Mapping[str, object] | None = None,
|
||||
realtime_mode: str = "realtime",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
|
|
@ -530,7 +857,24 @@ async def _arealtime(
|
|||
custom_llm_provider=_custom_llm_provider,
|
||||
)
|
||||
|
||||
deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs)
|
||||
session_litellm_metadata: Final = (
|
||||
dict(_translation_litellm_metadata(kwargs))
|
||||
if realtime_mode == "translation"
|
||||
else _build_litellm_metadata(kwargs)
|
||||
)
|
||||
translation_rejection: Final = (
|
||||
_translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, session_litellm_metadata)
|
||||
if realtime_mode == "translation"
|
||||
else None
|
||||
)
|
||||
if translation_rejection is not None:
|
||||
await _refuse_translation_session(
|
||||
websocket, translation_rejection, litellm_logging_obj, model, _custom_llm_provider
|
||||
)
|
||||
litellm_logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True
|
||||
return
|
||||
|
||||
deployment_params: Final = _OBJECT_MAPPING.validate_python(kwargs)
|
||||
if live_session_start is not None:
|
||||
await _alive_session(
|
||||
model=model,
|
||||
|
|
@ -540,7 +884,7 @@ async def _arealtime(
|
|||
session_start=live_session_start,
|
||||
api_base=dynamic_api_base or litellm_params.api_base,
|
||||
api_key=dynamic_api_key,
|
||||
litellm_metadata=_build_litellm_metadata(kwargs),
|
||||
litellm_metadata=session_litellm_metadata,
|
||||
litellm_params=deployment_params,
|
||||
)
|
||||
return
|
||||
|
|
@ -566,7 +910,7 @@ async def _arealtime(
|
|||
timeout=timeout,
|
||||
headers=headers,
|
||||
user_api_key_dict=kwargs.get("user_api_key_dict"),
|
||||
litellm_metadata=_build_litellm_metadata(kwargs),
|
||||
litellm_metadata=session_litellm_metadata,
|
||||
query_params=query_params,
|
||||
)
|
||||
elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
|
||||
|
|
@ -581,9 +925,6 @@ async def _arealtime(
|
|||
or litellm_params.get("realtime_protocol")
|
||||
or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL")
|
||||
)
|
||||
realtime_protocol: Final = azure_realtime_protocol_for_client(
|
||||
configured_realtime_protocol, query_params=query_params, websocket=websocket
|
||||
)
|
||||
resolved_azure_ad_token: Final = (
|
||||
None
|
||||
if api_key
|
||||
|
|
@ -592,6 +933,14 @@ async def _arealtime(
|
|||
GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token),
|
||||
)
|
||||
)
|
||||
realtime_protocol: Final = azure_realtime_protocol_for_client(
|
||||
configured_realtime_protocol,
|
||||
model=model,
|
||||
base_model=_deployment_base_model(kwargs, _custom_llm_provider),
|
||||
query_params=query_params,
|
||||
realtime_mode=realtime_mode,
|
||||
websocket=websocket,
|
||||
)
|
||||
await azure_realtime.async_realtime(
|
||||
model=model,
|
||||
websocket=websocket,
|
||||
|
|
@ -602,10 +951,13 @@ async def _arealtime(
|
|||
client=None,
|
||||
timeout=timeout,
|
||||
logging_obj=litellm_logging_obj,
|
||||
realtime_protocol=realtime_protocol,
|
||||
realtime_protocol=realtime_protocol.name,
|
||||
query_params=query_params,
|
||||
realtime_mode=realtime_mode,
|
||||
user_api_key_dict=kwargs.get("user_api_key_dict"),
|
||||
litellm_metadata=_build_litellm_metadata(kwargs),
|
||||
litellm_metadata=session_litellm_metadata,
|
||||
bridges_beta_client_to_ga=realtime_protocol.bridges_beta_client_to_ga,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
elif _custom_llm_provider == "openai":
|
||||
api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/"
|
||||
|
|
@ -622,8 +974,10 @@ async def _arealtime(
|
|||
client=None,
|
||||
timeout=timeout,
|
||||
query_params=query_params,
|
||||
realtime_mode=realtime_mode,
|
||||
user_api_key_dict=kwargs.get("user_api_key_dict"),
|
||||
litellm_metadata=_build_litellm_metadata(kwargs),
|
||||
litellm_metadata=session_litellm_metadata,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
elif _custom_llm_provider == "bedrock":
|
||||
# Extract AWS parameters from kwargs
|
||||
|
|
@ -678,7 +1032,7 @@ async def _arealtime(
|
|||
timeout=timeout,
|
||||
query_params=query_params,
|
||||
user_api_key_dict=kwargs.get("user_api_key_dict"),
|
||||
litellm_metadata=_build_litellm_metadata(kwargs),
|
||||
litellm_metadata=session_litellm_metadata,
|
||||
)
|
||||
elif _custom_llm_provider == "vertex_ai":
|
||||
vertex_credentials: Final = (
|
||||
|
|
@ -738,7 +1092,7 @@ async def _arealtime(
|
|||
timeout=timeout,
|
||||
headers=headers,
|
||||
user_api_key_dict=kwargs.get("user_api_key_dict"),
|
||||
litellm_metadata=_build_litellm_metadata(kwargs),
|
||||
litellm_metadata=session_litellm_metadata,
|
||||
query_params=query_params,
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -832,9 +832,11 @@ def _with_router_resolved_session_model(session: object, model_name: str) -> Map
|
|||
typed_session: Final = _SESSION_ADAPTER.validate_python(session)
|
||||
except ValidationError:
|
||||
return _NO_SESSION_KWARGS
|
||||
if "model" not in typed_session:
|
||||
if "model" not in typed_session and typed_session.get("type") != "transcription":
|
||||
return _NO_SESSION_KWARGS
|
||||
return MappingProxyType({"session": {**typed_session, "model": model_name}})
|
||||
from litellm.realtime_api.main import with_resolved_session_model
|
||||
|
||||
return MappingProxyType({"session": with_resolved_session_model(dict(typed_session), model_name)})
|
||||
|
||||
|
||||
# Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks
|
||||
|
|
@ -2850,6 +2852,14 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._host.invoke("acreate_realtime_transcription_session"),
|
||||
call_type="acreate_realtime_transcription_session",
|
||||
)
|
||||
self.acreate_realtime_translation_client_secret = self.factory_function(
|
||||
self._host.invoke("acreate_realtime_translation_client_secret"),
|
||||
call_type="acreate_realtime_translation_client_secret",
|
||||
)
|
||||
self.arealtime_translation_calls = self.factory_function(
|
||||
self._host.invoke("arealtime_translation_calls"),
|
||||
call_type="arealtime_translation_calls",
|
||||
)
|
||||
self._aresponses_websocket = self.factory_function(
|
||||
self._host.invoke("_aresponses_websocket"), call_type="_aresponses_websocket"
|
||||
)
|
||||
|
|
@ -8406,6 +8416,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"acreate_realtime_client_secret",
|
||||
"arealtime_calls",
|
||||
"acreate_realtime_transcription_session",
|
||||
"acreate_realtime_translation_client_secret",
|
||||
"arealtime_translation_calls",
|
||||
"_aresponses_websocket",
|
||||
"acreate_fine_tuning_job",
|
||||
"acancel_fine_tuning_job",
|
||||
|
|
@ -8676,6 +8688,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
"acreate_realtime_client_secret",
|
||||
"arealtime_calls",
|
||||
"acreate_realtime_transcription_session",
|
||||
"acreate_realtime_translation_client_secret",
|
||||
"arealtime_translation_calls",
|
||||
):
|
||||
return await self._ageneric_api_call_with_fallbacks(
|
||||
original_function=original_function,
|
||||
|
|
|
|||
|
|
@ -1236,9 +1236,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]]
|
|||
|
||||
OpenAIAudioTranscriptionOptionalParams = Literal[
|
||||
"language",
|
||||
"languages",
|
||||
"keywords",
|
||||
"prompt",
|
||||
"temperature",
|
||||
"response_format",
|
||||
"stream",
|
||||
"timestamp_granularities",
|
||||
"include",
|
||||
]
|
||||
|
|
@ -2511,6 +2514,17 @@ class OpenAIRealtimeResponseUsage(TypedDict):
|
|||
output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]]
|
||||
|
||||
|
||||
class OpenAIRealtimeTranslationDurationUsage(TypedDict):
|
||||
type: ReadOnly[Literal["duration"]]
|
||||
output_seconds: NotRequired[ReadOnly[float]]
|
||||
input_seconds: NotRequired[ReadOnly[float]]
|
||||
|
||||
|
||||
class OpenAIRealtimeTranslationClosedEvent(TypedDict):
|
||||
type: ReadOnly[Literal["session.closed"]]
|
||||
usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage]
|
||||
|
||||
|
||||
class OpenAIRealtimeEventTypes(Enum):
|
||||
SESSION_CREATED = "session.created"
|
||||
# Beta delta event names
|
||||
|
|
@ -2554,6 +2568,7 @@ OpenAIRealtimeEvents = (
|
|||
| OpenAIRealtimeInputAudioTranscriptionCompleted
|
||||
| OpenAIRealtimeTranscriptionSessionCreated
|
||||
| OpenAIRealtimeErrorEvent
|
||||
| OpenAIRealtimeTranslationClosedEvent
|
||||
)
|
||||
|
||||
OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import ConfigDict
|
||||
|
|
@ -66,6 +67,41 @@ class RealtimeExpiresAfter(LiteLLMBaseModel):
|
|||
seconds: int | None = None
|
||||
|
||||
|
||||
class RealtimeAudioTranscriptionConfig(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
model: str | None = None
|
||||
delay: str | None = None
|
||||
keywords: Sequence[str] | None = None
|
||||
language: str | None = None
|
||||
languages: Sequence[str] | None = None
|
||||
prompt: str | None = None
|
||||
|
||||
|
||||
class RealtimeAudioInputConfig(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
format: str | Mapping[str, object] | None = None
|
||||
noise_reduction: Mapping[str, object] | None = None
|
||||
transcription: RealtimeAudioTranscriptionConfig | None = None
|
||||
turn_detection: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class RealtimeAudioOutputConfig(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
format: str | Mapping[str, object] | None = None
|
||||
language: str | None = None
|
||||
voice: str | Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class RealtimeSessionAudioConfig(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
input: RealtimeAudioInputConfig | None = None
|
||||
output: RealtimeAudioOutputConfig | None = None
|
||||
|
||||
|
||||
class RealtimeSessionConfig(LiteLLMBaseModel):
|
||||
"""
|
||||
Session configuration nested inside the client_secrets request body.
|
||||
|
|
@ -75,12 +111,12 @@ class RealtimeSessionConfig(LiteLLMBaseModel):
|
|||
Extra/unknown fields are passed through unchanged.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
type: str | None = None
|
||||
model: str | None = None
|
||||
instructions: str | None = None
|
||||
audio: dict[str, object] | None = None
|
||||
audio: RealtimeSessionAudioConfig | None = None
|
||||
include: list[str] | None = None
|
||||
max_output_tokens: int | str | None = None
|
||||
output_modalities: list[str] | None = None
|
||||
|
|
@ -129,17 +165,20 @@ class RealtimeTranscriptionSessionRequest(LiteLLMBaseModel):
|
|||
unchanged to the provider.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
# LiteLLM-only routing hint — stripped before forwarding upstream.
|
||||
model: str | None = None
|
||||
input_audio_transcription: dict[str, Any] | None = None
|
||||
audio: RealtimeSessionAudioConfig | None = None
|
||||
|
||||
def resolved_model(self) -> str | None:
|
||||
if self.model:
|
||||
return self.model
|
||||
if self.input_audio_transcription:
|
||||
return self.input_audio_transcription.get("model")
|
||||
if self.audio and self.audio.input and self.audio.input.transcription:
|
||||
return self.audio.input.transcription.model
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -151,7 +190,7 @@ class RealtimeTranscriptionSessionResponse(LiteLLMBaseModel):
|
|||
ephemeral key. Unknown fields pass through unchanged.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
client_secret: dict[str, object] | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -550,6 +550,11 @@ class CallTypes(str, Enum):
|
|||
aextraction = "aextraction"
|
||||
arealtime = "_arealtime"
|
||||
aresponses_websocket = "_aresponses_websocket"
|
||||
acreate_realtime_client_secret = "acreate_realtime_client_secret"
|
||||
arealtime_calls = "arealtime_calls"
|
||||
acreate_realtime_transcription_session = "acreate_realtime_transcription_session"
|
||||
acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret"
|
||||
arealtime_translation_calls = "arealtime_translation_calls"
|
||||
create_batch = "create_batch"
|
||||
acreate_batch = "acreate_batch"
|
||||
aretrieve_batch = "aretrieve_batch"
|
||||
|
|
@ -790,6 +795,8 @@ CallTypesLiteral = Literal[
|
|||
"acreate_realtime_client_secret",
|
||||
"arealtime_calls",
|
||||
"acreate_realtime_transcription_session",
|
||||
"acreate_realtime_translation_client_secret",
|
||||
"arealtime_translation_calls",
|
||||
]
|
||||
|
||||
MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset(
|
||||
|
|
@ -798,6 +805,10 @@ MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset(
|
|||
|
||||
# Mapping of API routes to their corresponding call types
|
||||
API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
|
||||
"/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,),
|
||||
"/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,),
|
||||
"/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,),
|
||||
"/realtime/translations/calls": (CallTypes.arealtime_translation_calls,),
|
||||
# Chat Completions
|
||||
"/chat/completions": [CallTypes.acompletion, CallTypes.completion],
|
||||
"/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion],
|
||||
|
|
@ -1116,9 +1127,11 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
|
|||
"/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
|
||||
"/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
|
||||
# Realtime API
|
||||
"/realtime": [CallTypes.arealtime],
|
||||
"/v1/realtime": [CallTypes.arealtime],
|
||||
"/openai/v1/realtime": [CallTypes.arealtime],
|
||||
"/realtime": (CallTypes.arealtime,),
|
||||
"/v1/realtime": (CallTypes.arealtime,),
|
||||
"/openai/v1/realtime": (CallTypes.arealtime,),
|
||||
"/realtime/translations": (CallTypes.arealtime,),
|
||||
"/v1/realtime/translations": (CallTypes.arealtime,),
|
||||
"/live/sessions": (CallTypes.arealtime,),
|
||||
"/v1/live/sessions": (CallTypes.arealtime,),
|
||||
"/openai/v1/live/sessions": (CallTypes.arealtime,),
|
||||
|
|
@ -2947,9 +2960,19 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel):
|
|||
input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None
|
||||
|
||||
|
||||
class TranscriptionDetectedLanguage(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
code: str
|
||||
|
||||
|
||||
_DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, ...])
|
||||
|
||||
|
||||
class TranscriptionResponse(OpenAIObject):
|
||||
text: str | None = None
|
||||
usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None
|
||||
languages: Sequence[TranscriptionDetectedLanguage] | None = None
|
||||
|
||||
_hidden_params: dict = {}
|
||||
_response_headers: dict | None = None
|
||||
|
|
@ -2962,8 +2985,28 @@ class TranscriptionResponse(OpenAIObject):
|
|||
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
|
||||
self._hidden_params = hidden_params
|
||||
|
||||
def __init__(self, text=None) -> None:
|
||||
super().__init__(text=text)
|
||||
def __init__(
|
||||
self,
|
||||
text: str | None = None,
|
||||
usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | Mapping[str, object] | None = None,
|
||||
languages: Sequence[TranscriptionDetectedLanguage | Mapping[str, object]] | None = None,
|
||||
**kwargs: object, # kwargs-ok: OpenAI-compatible responses preserve provider extension fields
|
||||
) -> None:
|
||||
super().__init__(
|
||||
text=text, usage=usage, languages=TranscriptionResponse.detected_languages(languages), **kwargs
|
||||
)
|
||||
if self.languages is None:
|
||||
del self.languages
|
||||
|
||||
@staticmethod
|
||||
def detected_languages(value: object) -> tuple[TranscriptionDetectedLanguage, ...] | None:
|
||||
try:
|
||||
return _DETECTED_LANGUAGES.validate_python(value, from_attributes=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
def set_audio_transcription_duration(self, duration: float) -> None:
|
||||
self._hidden_params["audio_transcription_duration"] = duration
|
||||
|
||||
def __contains__(self, key) -> bool:
|
||||
# Define custom behavior for the 'in' operator
|
||||
|
|
|
|||
162
litellm/utils.py
162
litellm/utils.py
|
|
@ -1115,6 +1115,40 @@ def _is_streaming_response_for_correlation(result: object) -> bool:
|
|||
return isinstance(result, CustomStreamWrapper)
|
||||
|
||||
|
||||
def _returns_requested_stream(kwargs: dict[str, object], call_type: CallTypes | str, result: object) -> bool:
|
||||
if call_type in (CallTypes.transcription, CallTypes.atranscription):
|
||||
return isinstance(result, (openai.Stream, openai.AsyncStream))
|
||||
return _is_streaming_request(kwargs=kwargs, call_type=call_type)
|
||||
|
||||
|
||||
def _transcription_runs_unstreamed(
|
||||
call_type: str, requested_kwargs: Mapping[str, object], call_kwargs: Mapping[str, object]
|
||||
) -> bool:
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
|
||||
|
||||
return (
|
||||
call_type == CallTypes.atranscription.value
|
||||
and transcription_stream_requested(requested_kwargs.get("stream"))
|
||||
and call_kwargs.get("stream") is False
|
||||
)
|
||||
|
||||
|
||||
def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> dict[str, object]:
|
||||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming
|
||||
|
||||
return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True)
|
||||
|
||||
|
||||
def _marked_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object:
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import TRANSCRIPTION_RAN_UNSTREAMED
|
||||
|
||||
if not runs_unstreamed or not isinstance(result, TranscriptionResponse):
|
||||
return result
|
||||
marked: Final = result.model_copy()
|
||||
marked.hidden_params = {**result.hidden_params, TRANSCRIPTION_RAN_UNSTREAMED: True}
|
||||
return marked
|
||||
|
||||
|
||||
def _is_converted_stream_result(result: object) -> bool:
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
||||
|
|
@ -1550,7 +1584,6 @@ def function_setup(
|
|||
applied_guardrails=applied_guardrails,
|
||||
supports_correlation_logging=is_async_call,
|
||||
)
|
||||
|
||||
## check if metadata is passed in
|
||||
litellm_params: Final[dict[str, object]] = {"api_base": ""}
|
||||
if "metadata" in kwargs:
|
||||
|
|
@ -1798,7 +1831,28 @@ async def async_pre_call_deployment_hook(kwargs: Mapping[str, object], call_type
|
|||
if result is not None:
|
||||
modified_kwargs = result
|
||||
|
||||
return modified_kwargs
|
||||
return _unstreamed_when_post_call_guardrails_apply(modified_kwargs, typed_call_type)
|
||||
|
||||
|
||||
def _unstreamed_when_post_call_guardrails_apply(
|
||||
kwargs: dict[str, object], call_type: CallTypes | None
|
||||
) -> dict[str, object]:
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
|
||||
|
||||
if (
|
||||
call_type is not CallTypes.atranscription
|
||||
or not transcription_stream_requested(kwargs.get("stream"))
|
||||
or _deployment_provider(kwargs) not in (LlmProviders.OPENAI.value, LlmProviders.AZURE.value)
|
||||
):
|
||||
return kwargs
|
||||
streamed_request: Final = _guardrail_view_of_requested_stream(kwargs)
|
||||
custom_guardrail_class: Final = _get_cached_custom_guardrail()
|
||||
if not any(
|
||||
isinstance(callback, custom_guardrail_class) and callback.runs_post_call_deployment_hook(streamed_request)
|
||||
for callback in litellm.callbacks
|
||||
):
|
||||
return kwargs
|
||||
return {**kwargs, "stream": False}
|
||||
|
||||
|
||||
async def async_post_call_success_deployment_hook(
|
||||
|
|
@ -2146,7 +2200,7 @@ def client(original_function):
|
|||
# MODEL CALL
|
||||
result = original_function(*args, **kwargs)
|
||||
end_time = datetime.datetime.now()
|
||||
if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result):
|
||||
if _returns_requested_stream(kwargs, call_type, result) or _is_converted_stream_result(result):
|
||||
if "complete_response" in kwargs and kwargs["complete_response"] is True:
|
||||
chunks = []
|
||||
for idx, chunk in enumerate(result):
|
||||
|
|
@ -2155,6 +2209,15 @@ def client(original_function):
|
|||
else:
|
||||
logging_obj.stream = True
|
||||
logging_obj.model_call_details["stream"] = True
|
||||
if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream):
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
litellm_exception_mapper,
|
||||
wrap_transcription_stream,
|
||||
)
|
||||
|
||||
result = wrap_transcription_stream(
|
||||
result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details)
|
||||
)
|
||||
# RETURN RESULT
|
||||
update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata
|
||||
update_response_metadata(
|
||||
|
|
@ -2330,6 +2393,7 @@ def client(original_function):
|
|||
|
||||
kwargs["litellm_logging_obj"] = logging_obj
|
||||
modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type)
|
||||
transcription_runs_unstreamed: Final = _transcription_runs_unstreamed(call_type, kwargs, modified_kwargs)
|
||||
if modified_kwargs is not None:
|
||||
kwargs = modified_kwargs
|
||||
|
||||
|
|
@ -2380,7 +2444,9 @@ def client(original_function):
|
|||
if _is_converted_stream_result(_caching_handler_response.cached_result):
|
||||
logging_obj.stream = True
|
||||
logging_obj.model_call_details["stream"] = True
|
||||
return _caching_handler_response.cached_result
|
||||
return _marked_when_run_unstreamed(
|
||||
_caching_handler_response.cached_result, transcription_runs_unstreamed
|
||||
)
|
||||
|
||||
elif _caching_handler_response.embedding_all_elements_cache_hit is True:
|
||||
return _caching_handler_response.final_embedding_cached_response
|
||||
|
|
@ -2443,7 +2509,7 @@ def client(original_function):
|
|||
raise
|
||||
end_time = datetime.datetime.now()
|
||||
|
||||
streaming_requested: Final = _is_streaming_request(kwargs=kwargs, call_type=call_type)
|
||||
streaming_requested: Final = _returns_requested_stream(kwargs, call_type, result)
|
||||
if streaming_requested or _is_converted_stream_result(result):
|
||||
logging_obj.stream = True
|
||||
logging_obj.model_call_details["stream"] = True
|
||||
|
|
@ -2457,6 +2523,15 @@ def client(original_function):
|
|||
chunks.append(chunk)
|
||||
return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None))
|
||||
else:
|
||||
if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream):
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
litellm_exception_mapper,
|
||||
wrap_transcription_stream,
|
||||
)
|
||||
|
||||
result = wrap_transcription_stream(
|
||||
result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details)
|
||||
)
|
||||
_update_response_metadata(
|
||||
result=result,
|
||||
logging_obj=logging_obj,
|
||||
|
|
@ -2482,10 +2557,15 @@ def client(original_function):
|
|||
# Only run if call_type is a valid value in CallTypes
|
||||
_call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type)
|
||||
if _call_type_enum is not None:
|
||||
result = await async_post_call_success_deployment_hook(
|
||||
request_data=call_kwargs,
|
||||
response=result,
|
||||
call_type=_call_type_enum,
|
||||
result = _marked_when_run_unstreamed(
|
||||
await async_post_call_success_deployment_hook(
|
||||
request_data=_guardrail_view_of_requested_stream(call_kwargs)
|
||||
if transcription_runs_unstreamed
|
||||
else call_kwargs,
|
||||
response=result,
|
||||
call_type=_call_type_enum,
|
||||
),
|
||||
transcription_runs_unstreamed,
|
||||
)
|
||||
|
||||
## Add response to cache
|
||||
|
|
@ -4316,40 +4396,61 @@ def _get_non_default_params(passed_params: dict, default_params: dict, additiona
|
|||
return non_default_params
|
||||
|
||||
|
||||
_OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS: Final = frozenset(("languages", "keywords", "stream"))
|
||||
|
||||
|
||||
def get_optional_params_transcription(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
language: str | None = None,
|
||||
languages: Sequence[str] | None = None,
|
||||
keywords: Sequence[str] | None = None,
|
||||
prompt: str | None = None,
|
||||
response_format: str | None = None,
|
||||
temperature: int | None = None,
|
||||
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
|
||||
stream: bool | None = None,
|
||||
drop_params: bool | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS
|
||||
|
||||
# retrieve all parameters passed to the function
|
||||
passed_params: Final = locals()
|
||||
passed_params: Final = locals().copy()
|
||||
|
||||
passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS")
|
||||
passed_params.pop("custom_llm_provider")
|
||||
passed_params.pop("drop_params")
|
||||
drop_params = normalize_drop_params(drop_params)
|
||||
passed_params.pop("model")
|
||||
custom_llm_provider = passed_params.pop("custom_llm_provider")
|
||||
drop_params = normalize_drop_params(passed_params.pop("drop_params"))
|
||||
special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs")
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
transcription_stream_flag,
|
||||
transcription_stream_unset,
|
||||
)
|
||||
|
||||
for k, v in special_params.items():
|
||||
passed_params[k] = v
|
||||
|
||||
default_params: Final = {
|
||||
"language": None,
|
||||
"prompt": None,
|
||||
"response_format": None,
|
||||
"temperature": None, # openai defaults this to 0
|
||||
"timestamp_granularities": None,
|
||||
}
|
||||
sends_through_openai_sdk: Final = custom_llm_provider in ("openai", "azure")
|
||||
drops_unsupported_params: Final = drop_params is True or litellm.drop_params is True
|
||||
transcription_params: Final = (
|
||||
OPENAI_TRANSCRIPTION_PARAMS
|
||||
if sends_through_openai_sdk or drops_unsupported_params
|
||||
else [param for param in OPENAI_TRANSCRIPTION_PARAMS if param not in _OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS]
|
||||
)
|
||||
default_params: Final = {param: None for param in (*transcription_params, "prompt", "temperature")}
|
||||
stream_unset: Final = transcription_stream_unset(stream)
|
||||
malformed_stream: Final = not stream_unset and transcription_stream_flag(stream) is None
|
||||
if malformed_stream and not drops_unsupported_params:
|
||||
raise UnsupportedParamsError(
|
||||
status_code=400,
|
||||
message="`stream` must be a boolean. To drop it from the call, set `litellm.drop_params = True`.",
|
||||
)
|
||||
|
||||
non_default_params: Final = {
|
||||
k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])
|
||||
k: v
|
||||
for k, v in passed_params.items()
|
||||
if k in default_params and v != default_params[k] and not (k == "stream" and (stream_unset or malformed_stream))
|
||||
}
|
||||
optional_params = {}
|
||||
|
||||
|
|
@ -4358,9 +4459,7 @@ def get_optional_params_transcription(
|
|||
if len(non_default_params.keys()) > 0:
|
||||
keys: Final = list(non_default_params.keys())
|
||||
for k in keys:
|
||||
if (
|
||||
drop_params is True or litellm.drop_params is True
|
||||
) and k not in supported_params: # drop the unsupported non-default values
|
||||
if drops_unsupported_params and k not in supported_params:
|
||||
non_default_params.pop(k, None)
|
||||
elif k not in supported_params:
|
||||
raise UnsupportedParamsError(
|
||||
|
|
@ -4414,9 +4513,12 @@ def get_optional_params_transcription(
|
|||
optional_params=optional_params,
|
||||
passed_params=passed_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS,
|
||||
openai_params=list(default_params) if custom_llm_provider == "bedrock" else transcription_params,
|
||||
additional_drop_params=kwargs.get("additional_drop_params", None),
|
||||
)
|
||||
extra_body: Final = optional_params.get("extra_body")
|
||||
if isinstance(extra_body, dict) and not extra_body:
|
||||
optional_params.pop("extra_body")
|
||||
|
||||
return optional_params
|
||||
|
||||
|
|
@ -7152,6 +7254,7 @@ def get_model_info_helper(
|
|||
"cache_read_input_audio_token_cost_batches", None
|
||||
),
|
||||
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
|
||||
cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
"cache_read_input_token_cost_above_200k_tokens", None
|
||||
|
|
@ -10178,6 +10281,7 @@ class ProviderConfigManager:
|
|||
def get_provider_audio_transcription_config(
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
base_model: str | None = None,
|
||||
) -> BaseAudioTranscriptionConfig | None:
|
||||
model_cost_entry: Final = _get_model_cost_entry_for_provider_config(
|
||||
model=model,
|
||||
|
|
@ -10205,7 +10309,13 @@ class ProviderConfigManager:
|
|||
|
||||
return XAIAudioTranscriptionConfig()
|
||||
elif litellm.LlmProviders.OPENAI == provider:
|
||||
if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model):
|
||||
if "gpt-transcribe" in (model, base_model):
|
||||
from litellm.llms.openai.transcriptions.gpt_transformation import (
|
||||
OpenAIGPTTranscribeAudioTranscriptionConfig,
|
||||
)
|
||||
|
||||
return OpenAIGPTTranscribeAudioTranscriptionConfig()
|
||||
elif "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model):
|
||||
return litellm.OpenAIGPTAudioTranscriptionConfig()
|
||||
else:
|
||||
return litellm.OpenAIWhisperAudioTranscriptionConfig()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,168 @@
|
|||
{
|
||||
"azure/gpt-realtime-2.1-2026-07-07": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
"input_cost_per_image_token": 5e-06,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_image_token_cost": 5e-07,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"azure/gpt-realtime-2.1-mini-2026-07-07": {
|
||||
"cache_creation_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"input_cost_per_audio_token": 1e-05,
|
||||
"input_cost_per_image_token": 8e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_image_token_cost": 8e-08,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"azure/gpt-realtime-translate-2026-05-06": {
|
||||
"input_cost_per_second": 0.0005666666666666667,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio",
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-realtime-translate-2026-05-07": {
|
||||
"input_cost_per_second": 0.0005666666666666667,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://ai.azure.com/catalog/models/gpt-realtime-translate",
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio",
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-realtime-whisper-2026-05-06": {
|
||||
"input_cost_per_second": 0.0002833333333333333,
|
||||
"litellm_provider": "azure",
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/transcription_sessions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096
|
||||
},
|
||||
"azure/gpt-realtime-whisper-2026-05-07": {
|
||||
"input_cost_per_second": 0.0002833333333333333,
|
||||
"litellm_provider": "azure",
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/transcription_sessions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096
|
||||
},
|
||||
"sample_spec": {
|
||||
"code_interpreter_cost_per_session": 0.0,
|
||||
"computer_use_input_cost_per_1k_tokens": 0.0,
|
||||
|
|
@ -6424,6 +6588,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6458,6 +6625,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6491,6 +6661,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6525,6 +6698,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6547,6 +6723,7 @@
|
|||
"azure/gpt-realtime-2.1": {
|
||||
"cache_creation_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"cache_read_input_image_token_cost": 5e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"deprecation_date": "2027-06-25",
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
|
|
@ -6556,6 +6733,9 @@
|
|||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
|
||||
|
|
@ -6575,12 +6755,15 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure/gpt-realtime-2.1-mini": {
|
||||
"cache_creation_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_audio_token_cost": 3e-07,
|
||||
"cache_read_input_image_token_cost": 8e-08,
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"deprecation_date": "2027-06-25",
|
||||
"input_cost_per_audio_token": 1e-05,
|
||||
|
|
@ -6590,6 +6773,9 @@
|
|||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
|
||||
|
|
@ -6609,6 +6795,8 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -6626,6 +6814,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6659,6 +6850,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 2e-05,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
],
|
||||
|
|
@ -6791,6 +6985,9 @@
|
|||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime",
|
||||
|
|
@ -29215,9 +29412,9 @@
|
|||
},
|
||||
"gemini-3-pro-image": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_flex": 1e-07,
|
||||
"cache_read_input_token_cost_priority": 3.6e-07,
|
||||
"deprecation_date": "2027-05-28",
|
||||
|
|
@ -39042,6 +39239,9 @@
|
|||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 32000,
|
||||
"mode": "realtime",
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
|
|
@ -39097,6 +39297,8 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -39133,6 +39335,8 @@
|
|||
"supports_audio_output": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -52544,9 +52748,9 @@
|
|||
},
|
||||
"vertex_ai/gemini-3-pro-image": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"cache_read_input_token_cost_flex": 1e-07,
|
||||
"cache_read_input_token_cost_priority": 3.6e-07,
|
||||
"deprecation_date": "2027-05-28",
|
||||
|
|
@ -63346,6 +63550,10 @@
|
|||
"input_cost_per_second": 7.5e-05,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {
|
||||
"transcription_json_or_text_only": 1,
|
||||
"transcription_usage_duration": 1
|
||||
},
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions",
|
||||
|
|
@ -63358,7 +63566,8 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"gpt-live-transcribe": {
|
||||
"input_cost_per_second": 0.000283333333333,
|
||||
|
|
@ -63376,7 +63585,8 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"gpt-live-1": {
|
||||
"input_cost_per_second": 0.000833333333333,
|
||||
|
|
@ -63410,7 +63620,8 @@
|
|||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true
|
||||
"supports_audio_output": true,
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"claude-mythos-5": {
|
||||
"supports_anthropic_compaction": true,
|
||||
|
|
@ -71245,6 +71456,9 @@
|
|||
"mode": "realtime",
|
||||
"output_cost_per_audio_token": 6.4e-05,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supported_endpoints": [
|
||||
"/v1/realtime"
|
||||
|
|
@ -71302,7 +71516,11 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-transcribe": {
|
||||
"deprecation_date": "2028-02-01",
|
||||
|
|
@ -71312,7 +71530,8 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions",
|
||||
"/v1/realtime/transcription_sessions"
|
||||
"/v1/realtime/transcription_sessions",
|
||||
"/v1/realtime"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -71321,7 +71540,13 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
"supports_audio_input": true,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1,
|
||||
"transcription_json_or_text_only": 1,
|
||||
"transcription_usage_duration": 1
|
||||
},
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"azure/gpt-realtime-translate": {
|
||||
"deprecation_date": "2027-11-09",
|
||||
|
|
@ -71339,7 +71564,11 @@
|
|||
"audio"
|
||||
],
|
||||
"supports_audio_input": true,
|
||||
"supports_audio_output": true
|
||||
"supports_audio_output": true,
|
||||
"provider_specific_entry": {
|
||||
"realtime_ga_only": 1
|
||||
},
|
||||
"supports_native_streaming": true
|
||||
},
|
||||
"aihubmix/agnes-2.5-flash": {
|
||||
"input_cost_per_token": 3e-08,
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from collections.abc import Iterator, Mapping
|
|||
from typing import Final, NamedTuple
|
||||
|
||||
GATED_METHODS: Final = frozenset({"post"})
|
||||
SCHEMA_MEDIA_TYPES: Final = ("application/json", "application/sdp")
|
||||
|
||||
|
||||
class Violation(NamedTuple):
|
||||
|
|
@ -30,18 +31,21 @@ def _as_mapping(node: object) -> Mapping[str, object]:
|
|||
return node if isinstance(node, Mapping) else {}
|
||||
|
||||
|
||||
def _json_schema(container: object) -> object:
|
||||
def _content_schema(container: object) -> object:
|
||||
content: Final = _as_mapping(_as_mapping(container).get("content"))
|
||||
return _as_mapping(content.get("application/json")).get("schema")
|
||||
return next(
|
||||
(schema for media_type in SCHEMA_MEDIA_TYPES if (schema := _as_mapping(content.get(media_type)).get("schema"))),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]:
|
||||
if not _json_schema(operation.get("requestBody")):
|
||||
if not _content_schema(operation.get("requestBody")):
|
||||
yield "OA001"
|
||||
description: Final = operation.get("description")
|
||||
if not isinstance(description, str) or not description.strip():
|
||||
yield "OA002"
|
||||
if not _json_schema(_as_mapping(operation.get("responses")).get("200")):
|
||||
if not _content_schema(_as_mapping(operation.get("responses")).get("200")):
|
||||
yield "OA003"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,10 @@
|
|||
- {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"}
|
||||
- {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"}
|
||||
- {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"}
|
||||
- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"}
|
||||
- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"}
|
||||
- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"}
|
||||
- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"}
|
||||
- {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"}
|
||||
- {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"}
|
||||
- {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"}
|
||||
|
|
@ -94,6 +98,8 @@
|
|||
- {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"}
|
||||
- {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"}
|
||||
- {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"}
|
||||
- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"}
|
||||
- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"}
|
||||
- {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"}
|
||||
- {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"}
|
||||
- {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"}
|
||||
|
|
|
|||
|
|
@ -93,6 +93,8 @@ LlmCapability = Literal[
|
|||
"tool_search_history",
|
||||
"tool_use",
|
||||
"upstream_stream_failure",
|
||||
"transcription",
|
||||
"translation",
|
||||
"vision",
|
||||
"web_search",
|
||||
"web_search_server_tool",
|
||||
|
|
@ -162,14 +164,7 @@ class OtherCell(_Base):
|
|||
|
||||
|
||||
Cell = Annotated[
|
||||
LlmCell
|
||||
| MgmtCell
|
||||
| McpCell
|
||||
| ReliabilityCell
|
||||
| QuotaCell
|
||||
| LoggingCell
|
||||
| GuardrailCell
|
||||
| OtherCell,
|
||||
LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell,
|
||||
Field(discriminator="module"),
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,15 @@ a matching `function_call` output item, the test sends a `function_call_output`
|
|||
back, and the follow-up response incorporates the result (the temperature 72
|
||||
appears). That raw-websocket tool path is the source of truth for tool calling.
|
||||
|
||||
The audio surfaces provision their own OpenAI and Azure deployments.
|
||||
`TestRealtimeTranslation` streams the spoken weather question into
|
||||
`/v1/realtime/translations` with French as the target and asserts translated
|
||||
audio, a French transcript, and a billed spend row. `TestRealtimeLiveTranscription`
|
||||
streams it into `/v1/realtime?intent=transcription` on gpt-live-transcribe and
|
||||
asserts the deltas add up to the completed transcript and the session is billed.
|
||||
`test_azure_ga_text_turn_is_answered_and_billed` guards a plain text turn over
|
||||
the Azure GA deployment, spend row included
|
||||
|
||||
`test_pipecat_tool_smoke` is a realism layer through pipecat for openai, azure,
|
||||
and gemini only (not vertex_ai: native-audio live is flaky under pipecat tool
|
||||
calling while raw-ws tools pass; see pipecat-ai/pipecat#2544). Assertions are
|
||||
|
|
|
|||
|
|
@ -29,9 +29,13 @@ from models import LiteLLMParamsBody
|
|||
|
||||
_M = TypeVar("_M", bound=BaseModel)
|
||||
|
||||
REALTIME_ROUTE: Final = "/v1/realtime"
|
||||
TRANSLATIONS_ROUTE: Final = "/v1/realtime/translations"
|
||||
|
||||
def realtime_ws_url(model: str) -> str:
|
||||
return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}"
|
||||
|
||||
def realtime_ws_url(model: str, *, route: str = REALTIME_ROUTE, intent: str | None = None) -> str:
|
||||
query: Final = {"model": model} if intent is None else {"model": model, "intent": intent}
|
||||
return f"{ws_base_url()}{route}?{urlencode(query)}"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -183,6 +187,75 @@ class InputAudioBufferAppend(BaseModel):
|
|||
audio: str
|
||||
|
||||
|
||||
class InputAudioBufferCommit(BaseModel):
|
||||
type: str = "input_audio_buffer.commit"
|
||||
|
||||
|
||||
class SessionInputAudioBufferAppend(BaseModel):
|
||||
type: str = "session.input_audio_buffer.append"
|
||||
audio: str
|
||||
|
||||
|
||||
AudioAppend = InputAudioBufferAppend | SessionInputAudioBufferAppend
|
||||
|
||||
|
||||
class TranslationOutput(BaseModel):
|
||||
language: str
|
||||
|
||||
|
||||
class TranslationAudio(BaseModel):
|
||||
output: TranslationOutput
|
||||
|
||||
|
||||
class TranslationSessionConfig(BaseModel):
|
||||
audio: TranslationAudio
|
||||
|
||||
|
||||
class TranslationSessionUpdate(BaseModel):
|
||||
type: str = "session.update"
|
||||
session: TranslationSessionConfig
|
||||
|
||||
|
||||
class PcmFormat(BaseModel):
|
||||
type: str = "audio/pcm"
|
||||
rate: int = 24000
|
||||
|
||||
|
||||
class InputTranscription(BaseModel):
|
||||
model: str
|
||||
|
||||
|
||||
class TranscriptionAudioInput(BaseModel):
|
||||
format: PcmFormat = PcmFormat()
|
||||
transcription: InputTranscription
|
||||
|
||||
|
||||
class TranscriptionAudio(BaseModel):
|
||||
input: TranscriptionAudioInput
|
||||
|
||||
|
||||
class TranscriptionSessionConfig(BaseModel):
|
||||
type: str = "transcription"
|
||||
audio: TranscriptionAudio
|
||||
|
||||
|
||||
class TranscriptionSessionUpdate(BaseModel):
|
||||
type: str = "session.update"
|
||||
session: TranscriptionSessionConfig
|
||||
|
||||
|
||||
RealtimeClientEvent = (
|
||||
SessionUpdate
|
||||
| TranslationSessionUpdate
|
||||
| TranscriptionSessionUpdate
|
||||
| ConversationItemCreate
|
||||
| ResponseCreate
|
||||
| InputAudioBufferAppend
|
||||
| InputAudioBufferCommit
|
||||
| SessionInputAudioBufferAppend
|
||||
)
|
||||
|
||||
|
||||
def user_message(text: str) -> ConversationItemCreate:
|
||||
return ConversationItemCreate(
|
||||
item=MessageItem(content=[InputTextContent(text=text)])
|
||||
|
|
@ -320,7 +393,7 @@ class RealtimeSession:
|
|||
connection: Connection
|
||||
|
||||
@step("Send the realtime event {event.type} over the websocket")
|
||||
def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate | InputAudioBufferAppend) -> None:
|
||||
def send(self, event: RealtimeClientEvent) -> None:
|
||||
self.connection.send(event.model_dump_json(by_alias=True, exclude_none=True))
|
||||
|
||||
@step("Wait for a {stop_type} event on the realtime websocket")
|
||||
|
|
@ -350,9 +423,9 @@ class RealtimeSession:
|
|||
@step("Stream the spoken question as mic chunks, then silence until the turn goes idle")
|
||||
def stream_and_collect(
|
||||
self,
|
||||
chunks: Sequence[InputAudioBufferAppend],
|
||||
chunks: Sequence[AudioAppend],
|
||||
*,
|
||||
tail: InputAudioBufferAppend,
|
||||
tail: AudioAppend,
|
||||
interval: float,
|
||||
idle: float,
|
||||
timeout: float,
|
||||
|
|
@ -363,9 +436,9 @@ class RealtimeSession:
|
|||
|
||||
def _live_mic_events(
|
||||
self,
|
||||
chunks: Sequence[InputAudioBufferAppend],
|
||||
chunks: Sequence[AudioAppend],
|
||||
*,
|
||||
tail: InputAudioBufferAppend,
|
||||
tail: AudioAppend,
|
||||
interval: float,
|
||||
idle: float,
|
||||
timeout: float,
|
||||
|
|
@ -408,13 +481,19 @@ class RealtimeClient:
|
|||
)
|
||||
return model_name, model_id
|
||||
|
||||
@step("Open a /v1/realtime websocket session to {model}")
|
||||
@step("Open a {route} websocket session to {model}")
|
||||
@contextmanager
|
||||
def connect(
|
||||
self, *, key: str, model: str, timeout: float = 15.0
|
||||
self,
|
||||
*,
|
||||
key: str,
|
||||
model: str,
|
||||
route: str = REALTIME_ROUTE,
|
||||
intent: str | None = None,
|
||||
timeout: float = 15.0,
|
||||
) -> Generator[RealtimeSession, None, None]:
|
||||
with connect(
|
||||
realtime_ws_url(model),
|
||||
realtime_ws_url(model, route=route, intent=intent),
|
||||
additional_headers={"Authorization": f"Bearer {key}"},
|
||||
open_timeout=timeout,
|
||||
) as connection:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@ Each test opens a websocket through the proxy, speaks the OpenAI GA realtime
|
|||
event schema, and asserts the proxy normalizes the provider's stream into that
|
||||
schema: the session lifecycle, the canonical response event sequence with a
|
||||
reconstructed transcript and usage, and a full tool-call round-trip (call ->
|
||||
tool result -> a follow-up response that uses the result).
|
||||
tool result -> a follow-up response that uses the result). The audio tests
|
||||
stream the spoken weather question into /v1/realtime/translations and into a
|
||||
/v1/realtime?intent=transcription session, and assert the translated or
|
||||
transcribed output and the session's spend row.
|
||||
|
||||
One GA-speaking client validates every provider; only the model alias changes.
|
||||
Every provider is provisioned at session start, so a missing realtime alias is a
|
||||
|
|
@ -12,19 +15,28 @@ hard failure, not a skip; once configured, a protocol failure is likewise a hard
|
|||
failure. See REALTIME_COVERAGE_MATRIX.md.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
from models import LiteLLMParamsBody, SpendLogRow
|
||||
from pydantic import BaseModel
|
||||
from realtime_client import (
|
||||
PROVIDERS,
|
||||
TRANSLATIONS_ROUTE,
|
||||
ConversationItemCreate,
|
||||
DeltaEvent,
|
||||
FunctionCallArgumentsDone,
|
||||
FunctionCallOutputItem,
|
||||
FunctionTool,
|
||||
InputAudioBufferAppend,
|
||||
InputAudioBufferCommit,
|
||||
InputTranscription,
|
||||
JsonSchema,
|
||||
JsonSchemaProperty,
|
||||
RealtimeClient,
|
||||
|
|
@ -35,8 +47,18 @@ from realtime_client import (
|
|||
ResponseDone,
|
||||
ServerEnvelope,
|
||||
SessionConfig,
|
||||
SessionInputAudioBufferAppend,
|
||||
SessionUpdate,
|
||||
TranscriptionAudio,
|
||||
TranscriptionAudioInput,
|
||||
TranscriptionSessionConfig,
|
||||
TranscriptionSessionUpdate,
|
||||
TranslationAudio,
|
||||
TranslationOutput,
|
||||
TranslationSessionConfig,
|
||||
TranslationSessionUpdate,
|
||||
as_text,
|
||||
events_of_type,
|
||||
parse_last,
|
||||
realtime_model,
|
||||
transcript,
|
||||
|
|
@ -264,3 +286,204 @@ def test_upstream_handshake_refusal_is_an_error_event_and_policy_close(
|
|||
|
||||
assert closed.value.rcvd is not None
|
||||
assert closed.value.rcvd.code == 1008, closed.value
|
||||
|
||||
|
||||
WEATHER_WAV: Final = Path(__file__).parent / "fixtures" / "weather_question_24k.wav"
|
||||
PCM_RATE: Final = 24000
|
||||
CHUNK_SECONDS: Final = 0.1
|
||||
CHUNK_BYTES: Final = int(PCM_RATE * CHUNK_SECONDS) * 2
|
||||
SILENCE_B64: Final = base64.b64encode(bytes(CHUNK_BYTES)).decode()
|
||||
REGISTRY_ROUTES: Final = MappingProxyType({"openai": "openai", "azure": "azure_openai"})
|
||||
|
||||
|
||||
def weather_pcm_chunks() -> tuple[str, ...]:
|
||||
with wave.open(str(WEATHER_WAV), "rb") as wav:
|
||||
assert (wav.getnchannels(), wav.getsampwidth(), wav.getframerate()) == (1, 2, PCM_RATE), (
|
||||
"fixture must be mono PCM16 at 24 kHz"
|
||||
)
|
||||
pcm: Final = wav.readframes(wav.getnframes())
|
||||
return tuple(base64.b64encode(pcm[i : i + CHUNK_BYTES]).decode() for i in range(0, len(pcm), CHUNK_BYTES))
|
||||
|
||||
|
||||
def has_billed_row(rows: list[SpendLogRow]) -> bool:
|
||||
return any((row.spend or 0) > 0 for row in rows)
|
||||
|
||||
|
||||
def audio_marks(provider: RealtimeProvider, capability: str) -> tuple[pytest.MarkDecorator, ...]:
|
||||
return (
|
||||
pytest.mark.covers(f"llm.realtime.{REGISTRY_ROUTES[provider.id]}.{capability}.stream.works"),
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.REALTIME,
|
||||
providers=(Provider(provider.id),),
|
||||
models=(provider.litellm_params.model,),
|
||||
capabilities=(Capability.AUDIO_INPUT,),
|
||||
mode=Mode.WEBSOCKET,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
TRANSLATION_PROVIDERS: Final = (
|
||||
RealtimeProvider(
|
||||
"openai",
|
||||
"openai-translate",
|
||||
LiteLLMParamsBody(model="openai/gpt-realtime-translate", api_key="os.environ/OPENAI_API_KEY"),
|
||||
),
|
||||
RealtimeProvider(
|
||||
"azure",
|
||||
"azure-translate",
|
||||
LiteLLMParamsBody(
|
||||
model="azure/gpt-realtime-translate",
|
||||
api_base="os.environ/AZURE_API_BASE",
|
||||
api_key="os.environ/AZURE_API_KEY",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
LIVE_TRANSCRIPTION_PROVIDERS: Final = (
|
||||
RealtimeProvider(
|
||||
"openai",
|
||||
"openai-live-transcribe",
|
||||
LiteLLMParamsBody(model="openai/gpt-live-transcribe", api_key="os.environ/OPENAI_API_KEY"),
|
||||
),
|
||||
RealtimeProvider(
|
||||
"azure",
|
||||
"azure-live-transcribe",
|
||||
LiteLLMParamsBody(
|
||||
model="azure/gpt-live-transcribe",
|
||||
api_base="os.environ/AZURE_API_BASE",
|
||||
api_key="os.environ/AZURE_API_KEY",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
TRANSLATION_PARAMS: Final = tuple(
|
||||
pytest.param(p, id=p.id, marks=audio_marks(p, "translation")) for p in TRANSLATION_PROVIDERS
|
||||
)
|
||||
LIVE_TRANSCRIPTION_PARAMS: Final = tuple(
|
||||
pytest.param(p, id=p.id, marks=audio_marks(p, "transcription")) for p in LIVE_TRANSCRIPTION_PROVIDERS
|
||||
)
|
||||
AZURE_GA_PROVIDER: Final = next(p for p in PROVIDERS if p.id == "azure")
|
||||
|
||||
|
||||
class TranscriptionCompleted(BaseModel):
|
||||
type: str
|
||||
transcript: str
|
||||
|
||||
|
||||
class TestRealtimeTranslation:
|
||||
@pytest.mark.parametrize("provider", TRANSLATION_PARAMS)
|
||||
def test_spoken_english_comes_back_as_french_audio_and_transcript_and_is_billed(
|
||||
self,
|
||||
client: RealtimeClient,
|
||||
resources: ResourceManager,
|
||||
scoped_key: str,
|
||||
provider: RealtimeProvider,
|
||||
) -> None:
|
||||
model, model_id = client.provision(provider)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
|
||||
with client.connect(key=scoped_key, model=model, route=TRANSLATIONS_ROUTE) as session:
|
||||
session.send(
|
||||
TranslationSessionUpdate(
|
||||
session=TranslationSessionConfig(audio=TranslationAudio(output=TranslationOutput(language="fr")))
|
||||
)
|
||||
)
|
||||
session.collect_until("session.updated", timeout=30)
|
||||
events: Final = session.stream_and_collect(
|
||||
tuple(SessionInputAudioBufferAppend(audio=chunk) for chunk in weather_pcm_chunks()),
|
||||
tail=SessionInputAudioBufferAppend(audio=SILENCE_B64),
|
||||
interval=CHUNK_SECONDS,
|
||||
idle=8,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
types: Final = tuple(e.type for e in events)
|
||||
audio_bytes: Final = sum(
|
||||
len(base64.b64decode(DeltaEvent.model_validate_json(e.payload).delta))
|
||||
for e in events_of_type(events, "session.output_audio.delta")
|
||||
)
|
||||
assert audio_bytes > 0, f"translation returned no translated audio; types={types}"
|
||||
translated: Final = "".join(
|
||||
DeltaEvent.model_validate_json(e.payload).delta
|
||||
for e in events_of_type(events, "session.output_transcript.delta")
|
||||
).lower()
|
||||
assert translated.strip(), f"translation returned no translated text; types={types}"
|
||||
assert "weather" not in translated, f"the spoken weather question came back untranslated: {translated!r}"
|
||||
|
||||
rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row)
|
||||
assert has_billed_row(rows), f"the translation session was not billed: {rows}"
|
||||
|
||||
|
||||
class TestRealtimeLiveTranscription:
|
||||
@pytest.mark.parametrize("provider", LIVE_TRANSCRIPTION_PARAMS)
|
||||
def test_spoken_question_streams_transcript_deltas_that_match_the_completed_transcript(
|
||||
self,
|
||||
client: RealtimeClient,
|
||||
resources: ResourceManager,
|
||||
scoped_key: str,
|
||||
provider: RealtimeProvider,
|
||||
) -> None:
|
||||
model, model_id = client.provision(provider)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
|
||||
with client.connect(key=scoped_key, model=model, intent="transcription") as session:
|
||||
session.send(
|
||||
TranscriptionSessionUpdate(
|
||||
session=TranscriptionSessionConfig(
|
||||
audio=TranscriptionAudio(
|
||||
input=TranscriptionAudioInput(transcription=InputTranscription(model=model))
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
session.collect_until("session.updated", timeout=30)
|
||||
for chunk in weather_pcm_chunks():
|
||||
session.send(InputAudioBufferAppend(audio=chunk))
|
||||
session.send(InputAudioBufferCommit())
|
||||
events: Final = session.collect_until("conversation.item.input_audio_transcription.completed", timeout=60)
|
||||
|
||||
completed: Final = TranscriptionCompleted.model_validate_json(events[-1].payload)
|
||||
assert "weather" in completed.transcript.lower(), (
|
||||
f"transcript of a spoken weather question does not mention weather: {completed.transcript!r}"
|
||||
)
|
||||
streamed: Final = "".join(
|
||||
DeltaEvent.model_validate_json(e.payload).delta
|
||||
for e in events_of_type(events, "conversation.item.input_audio_transcription.delta")
|
||||
)
|
||||
assert streamed.strip() == completed.transcript.strip(), (
|
||||
f"streamed deltas {streamed!r} do not add up to the completed transcript {completed.transcript!r}"
|
||||
)
|
||||
|
||||
rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row)
|
||||
assert has_billed_row(rows), f"the live transcription session was not billed: {rows}"
|
||||
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.REALTIME,
|
||||
providers=(Provider.AZURE,),
|
||||
models=(AZURE_GA_PROVIDER.litellm_params.model,),
|
||||
mode=Mode.WEBSOCKET,
|
||||
)
|
||||
)
|
||||
def test_azure_ga_text_turn_is_answered_and_billed(
|
||||
client: RealtimeClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model, model_id = client.provision(AZURE_GA_PROVIDER)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
|
||||
with client.connect(key=scoped_key, model=model) as session:
|
||||
session.collect_until("session.created", timeout=20)
|
||||
session.send(SessionUpdate(session=SessionConfig(instructions="Reply with the single word hello.")))
|
||||
session.collect_until("session.updated", timeout=20)
|
||||
session.send(user_message("Say the single word hello."))
|
||||
session.send(ResponseCreate())
|
||||
events: Final = session.collect_until("response.done", timeout=60)
|
||||
|
||||
assert "hello" in transcript(events).lower(), f"Azure GA realtime did not answer the turn: {transcript(events)!r}"
|
||||
rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row)
|
||||
assert has_billed_row(rows), f"the Azure GA realtime turn was not billed: {rows}"
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ the shared multipart transport, since the SDK refuses to send them. A model-less
|
|||
request comes back as one of two 400s depending on whether any wildcard
|
||||
deployment happens to be registered on the shared proxy, so the assertion
|
||||
accepts either phrasing and holds both to naming the model as the problem.
|
||||
Streamed gpt-transcribe (stream=true) on OpenAI and Azure must send transcript
|
||||
deltas, a done event, and data: [DONE], and bill the call.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -20,10 +23,10 @@ from e2e_config import unique_marker
|
|||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from e2e_http import UnknownApiError, unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
from models import LiteLLMParamsBody, SpendLogRow
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, Field
|
||||
from sdk_clients import SdkClients
|
||||
from sdk_clients import SdkClients, response_header
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
|
@ -221,3 +224,102 @@ class TestWhisperTranscriptionFormats:
|
|||
assert all(word.start <= word.end for word in transcript.words), (
|
||||
f"word timings out of order: {transcript.words}"
|
||||
)
|
||||
|
||||
|
||||
STREAM_DONE: Final = "[DONE]"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StreamingTranscriber:
|
||||
id: str
|
||||
registry_route: str
|
||||
provider: Provider
|
||||
litellm_params: LiteLLMParamsBody
|
||||
|
||||
|
||||
STREAMING_TRANSCRIBERS: Final = (
|
||||
_StreamingTranscriber(
|
||||
"openai",
|
||||
"openai",
|
||||
Provider.OPENAI,
|
||||
LiteLLMParamsBody(model="openai/gpt-transcribe", api_key="os.environ/OPENAI_API_KEY"),
|
||||
),
|
||||
_StreamingTranscriber(
|
||||
"azure",
|
||||
"azure_openai",
|
||||
Provider.AZURE,
|
||||
LiteLLMParamsBody(
|
||||
model="azure/gpt-transcribe",
|
||||
api_base="os.environ/AZURE_API_BASE",
|
||||
api_key="os.environ/AZURE_API_KEY",
|
||||
api_version="v1",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
STREAMING_PARAMS: Final = tuple(
|
||||
pytest.param(
|
||||
t,
|
||||
id=t.id,
|
||||
marks=(
|
||||
pytest.mark.covers(f"llm.audio_transcriptions.{t.registry_route}.transcription.stream.works"),
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.AUDIO,
|
||||
providers=(t.provider,),
|
||||
models=(t.litellm_params.model,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
for t in STREAMING_TRANSCRIBERS
|
||||
)
|
||||
|
||||
|
||||
class _TranscriptStreamEvent(BaseModel):
|
||||
type: str
|
||||
delta: str = ""
|
||||
text: str = ""
|
||||
|
||||
|
||||
def _has_billed_row(rows: list[SpendLogRow]) -> bool:
|
||||
return any((row.spend or 0) > 0 for row in rows)
|
||||
|
||||
|
||||
class TestStreamedAudioTranscriptions:
|
||||
@pytest.mark.parametrize("transcriber", STREAMING_PARAMS)
|
||||
def test_stream_sends_transcript_deltas_then_done_and_bills_the_call(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, transcriber: _StreamingTranscriber
|
||||
) -> None:
|
||||
model: Final = f"e2e-transcribe-stream-{transcriber.id}-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(model, transcriber.litellm_params)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
client: Final = sdk.openai(resources.key())
|
||||
|
||||
with client.audio.transcriptions.with_streaming_response.create(
|
||||
model=model, file=(WEATHER_WAV.name, WEATHER_WAV.read_bytes(), "audio/wav"), stream=True
|
||||
) as response:
|
||||
content_type: Final = response_header(response.headers, "content-type")
|
||||
call_id: Final = response_header(response.headers, "x-litellm-call-id")
|
||||
payloads: Final = tuple(
|
||||
line.removeprefix("data: ") for line in response.iter_lines() if line.startswith("data: ")
|
||||
)
|
||||
|
||||
assert "text/event-stream" in (content_type or ""), f"stream=true did not stream: {content_type!r}"
|
||||
assert payloads and payloads[-1] == STREAM_DONE, f"the stream did not end with data: [DONE]: {payloads[-3:]}"
|
||||
assert payloads.count(STREAM_DONE) == 1, f"data: [DONE] was sent more than once: {payloads}"
|
||||
events: Final = tuple(_TranscriptStreamEvent.model_validate_json(payload) for payload in payloads[:-1])
|
||||
deltas: Final = tuple(e.delta for e in events if e.type == "transcript.text.delta")
|
||||
assert len(deltas) > 1, f"the transcript arrived in {len(deltas)} delta events, not as a stream: {events}"
|
||||
done: Final = events[-1]
|
||||
assert done.type == "transcript.text.done", f"the last event before [DONE] is not the done event: {done}"
|
||||
assert "weather" in done.text.lower(), f"transcript of a spoken weather question lost the word: {done.text!r}"
|
||||
assert "".join(deltas).strip() == done.text.strip(), (
|
||||
f"the deltas {deltas} do not add up to the final transcript {done.text!r}"
|
||||
)
|
||||
|
||||
assert call_id, "the streamed response carries no x-litellm-call-id to find its spend log"
|
||||
rows: Final = proxy.poll_logs_for_request_id(call_id, predicate=_has_billed_row)
|
||||
assert _has_billed_row(rows), f"the streamed transcription was not billed: {rows}"
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
"""Vendor §9.19: realtime client_secrets + calls HTTP surface (LIT-4778).
|
||||
|
||||
Websocket coverage already lives under realtime/; this file pins the HTTP
|
||||
client-secret mint and the missing-auth contract.
|
||||
client-secret mint (realtime, translation and transcription sessions), the
|
||||
translation token scope, and the missing-auth contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import NoBody, assert_auth_denied, unwrap
|
||||
from e2e_http import NoBody, PartialBody, UnknownApiError, assert_auth_denied, unwrap
|
||||
from e2e_metadata import Domain, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
|
|
@ -18,6 +22,7 @@ from pydantic import BaseModel
|
|||
pytestmark = pytest.mark.e2e
|
||||
|
||||
REALTIME_BACKEND = "openai/gpt-realtime"
|
||||
TRANSLATIONS_CLIENT_SECRETS: Final = "/v1/realtime/translations/client_secrets"
|
||||
|
||||
|
||||
class RealtimeSession(BaseModel):
|
||||
|
|
@ -38,8 +43,65 @@ class RealtimeClientSecretRequest(BaseModel):
|
|||
session: RealtimeSession | None = None
|
||||
|
||||
|
||||
class TurnDetection(BaseModel):
|
||||
type: str
|
||||
|
||||
|
||||
class SessionTranscription(BaseModel):
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class SessionAudioInput(BaseModel):
|
||||
transcription: SessionTranscription | None = None
|
||||
turn_detection: TurnDetection | None = None
|
||||
|
||||
|
||||
class SessionAudioOutput(BaseModel):
|
||||
language: str | None = None
|
||||
|
||||
|
||||
class SessionAudio(BaseModel):
|
||||
input: SessionAudioInput | None = None
|
||||
output: SessionAudioOutput | None = None
|
||||
|
||||
|
||||
class TranslationSession(BaseModel):
|
||||
model: str
|
||||
audio: SessionAudio
|
||||
|
||||
|
||||
class TranscriptionSession(BaseModel):
|
||||
type: str = "transcription"
|
||||
audio: SessionAudio
|
||||
|
||||
|
||||
class SessionClientSecretRequest(BaseModel):
|
||||
session: TranslationSession | TranscriptionSession
|
||||
|
||||
|
||||
class NullableAudioInput(PartialBody):
|
||||
turn_detection: TurnDetection | None = None
|
||||
|
||||
|
||||
class NullableAudio(PartialBody):
|
||||
input: NullableAudioInput
|
||||
|
||||
|
||||
class NullableRealtimeSession(PartialBody):
|
||||
type: str
|
||||
model: str
|
||||
instructions: str | None = None
|
||||
audio: NullableAudio
|
||||
|
||||
|
||||
class NullableClientSecretRequest(PartialBody):
|
||||
model: str
|
||||
session: NullableRealtimeSession
|
||||
|
||||
|
||||
class RealtimeClientSecretSession(BaseModel):
|
||||
type: str | None = None
|
||||
audio: SessionAudio | None = None
|
||||
|
||||
|
||||
class RealtimeClientSecretResponse(BaseModel):
|
||||
|
|
@ -49,15 +111,85 @@ class RealtimeClientSecretResponse(BaseModel):
|
|||
|
||||
|
||||
def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
|
||||
model = f"e2e-realtime-http-{unique_marker()}"
|
||||
model_id = proxy.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY"),
|
||||
return _register_backend(
|
||||
proxy, resources, LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
|
||||
def _register_backend(
|
||||
proxy: ProxyClient, resources: ResourceManager, litellm_params: LiteLLMParamsBody
|
||||
) -> tuple[str, str]:
|
||||
model: Final = f"e2e-realtime-http-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(model, litellm_params)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model, resources.key()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AudioBackend:
|
||||
id: str
|
||||
registry_route: str
|
||||
provider: Provider
|
||||
litellm_params: LiteLLMParamsBody
|
||||
|
||||
@property
|
||||
def upstream_model(self) -> str:
|
||||
return self.litellm_params.model.split("/", 1)[1]
|
||||
|
||||
|
||||
def _openai_backend(model: str) -> _AudioBackend:
|
||||
return _AudioBackend(
|
||||
"openai", "openai", Provider.OPENAI, LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
|
||||
def _azure_backend(model: str) -> _AudioBackend:
|
||||
return _AudioBackend(
|
||||
"azure",
|
||||
"azure_openai",
|
||||
Provider.AZURE,
|
||||
LiteLLMParamsBody(model=model, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"),
|
||||
)
|
||||
|
||||
|
||||
TRANSLATION_BACKENDS: Final = (
|
||||
_openai_backend("openai/gpt-realtime-translate"),
|
||||
_azure_backend("azure/gpt-realtime-translate"),
|
||||
)
|
||||
TRANSCRIPTION_BACKENDS: Final = (
|
||||
_openai_backend("openai/gpt-live-transcribe"),
|
||||
_azure_backend("azure/gpt-live-transcribe"),
|
||||
)
|
||||
|
||||
|
||||
def _subject(backend: _AudioBackend) -> pytest.MarkDecorator:
|
||||
return meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.REALTIME,
|
||||
providers=(backend.provider,),
|
||||
models=(backend.litellm_params.model,),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
TRANSLATION_PARAMS: Final = tuple(
|
||||
pytest.param(
|
||||
b,
|
||||
id=b.id,
|
||||
marks=(pytest.mark.covers(f"llm.realtime.{b.registry_route}.translation.stream.works"), _subject(b)),
|
||||
)
|
||||
for b in TRANSLATION_BACKENDS
|
||||
)
|
||||
TRANSCRIPTION_PARAMS: Final = tuple(pytest.param(b, id=b.id, marks=_subject(b)) for b in TRANSCRIPTION_BACKENDS)
|
||||
|
||||
|
||||
def _french_translation_request(model: str) -> SessionClientSecretRequest:
|
||||
return SessionClientSecretRequest(
|
||||
session=TranslationSession(model=model, audio=SessionAudio(output=SessionAudioOutput(language="fr")))
|
||||
)
|
||||
|
||||
|
||||
class TestRealtimeHttp:
|
||||
@pytest.mark.covers("llm.realtime.openai.basic.nonstream.works")
|
||||
@meta(
|
||||
|
|
@ -110,3 +242,150 @@ class TestRealtimeHttp:
|
|||
json=NoBody(),
|
||||
)
|
||||
assert_auth_denied(result, "realtime calls missing auth")
|
||||
|
||||
|
||||
class TestRealtimeTranslationClientSecrets:
|
||||
@pytest.mark.parametrize("backend", TRANSLATION_PARAMS)
|
||||
def test_translations_route_mints_a_french_translation_session(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend
|
||||
) -> None:
|
||||
model, key = _register_backend(proxy, resources, backend.litellm_params)
|
||||
secret: Final = unwrap(
|
||||
proxy.transport.post(
|
||||
TRANSLATIONS_CLIENT_SECRETS,
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=_french_translation_request(model),
|
||||
response_type=RealtimeClientSecretResponse,
|
||||
)
|
||||
)
|
||||
assert secret.value, f"client secret value missing: {secret}"
|
||||
assert secret.session is not None and secret.session.type == "translation", (
|
||||
f"the translations route did not mint a translation session: {secret.session}"
|
||||
)
|
||||
output: Final = secret.session.audio.output if secret.session.audio else None
|
||||
assert output is not None and output.language == "fr", (
|
||||
f"the requested target language did not reach the session: {secret.session}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.PROXY_AUTH,
|
||||
route=Route.REALTIME,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(TRANSLATION_BACKENDS[0].litellm_params.model,),
|
||||
)
|
||||
)
|
||||
def test_translation_token_is_refused_by_the_realtime_calls_route(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params)
|
||||
secret: Final = unwrap(
|
||||
proxy.transport.post(
|
||||
TRANSLATIONS_CLIENT_SECRETS,
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=_french_translation_request(model),
|
||||
response_type=RealtimeClientSecretResponse,
|
||||
)
|
||||
)
|
||||
assert secret.value, f"client secret value missing: {secret}"
|
||||
result: Final = proxy.transport.send(
|
||||
"/v1/realtime/calls", headers=proxy.transport.bearer(secret.value), json=NoBody()
|
||||
)
|
||||
assert result.status_code == 401, (
|
||||
f"a translation token opened a plain realtime call: {result.status_code} {result.body[:300]}"
|
||||
)
|
||||
assert "not valid for this Realtime endpoint" in result.body, (
|
||||
f"the refusal does not say the token belongs to another endpoint: {result.body[:300]}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.REALTIME,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(TRANSLATION_BACKENDS[0].litellm_params.model,),
|
||||
)
|
||||
)
|
||||
def test_translation_session_on_the_realtime_route_is_rejected(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params)
|
||||
result: Final = proxy.transport.post(
|
||||
"/v1/realtime/client_secrets",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=RealtimeClientSecretRequest(model=model, session=RealtimeSession(type="translation", model=model)),
|
||||
response_type=RealtimeClientSecretResponse,
|
||||
)
|
||||
match result:
|
||||
case UnknownApiError(status_code=400, body=body):
|
||||
assert "translations endpoint" in body, f"the 400 does not point at the translations route: {body}"
|
||||
case other:
|
||||
pytest.fail(f"a translation session on /v1/realtime/client_secrets expected a 400, got {other!r}")
|
||||
|
||||
|
||||
class TestRealtimeTranscriptionClientSecrets:
|
||||
@pytest.mark.parametrize("backend", TRANSCRIPTION_PARAMS)
|
||||
def test_transcription_session_runs_on_the_deployment_model(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend
|
||||
) -> None:
|
||||
model, key = _register_backend(proxy, resources, backend.litellm_params)
|
||||
secret: Final = unwrap(
|
||||
proxy.transport.post(
|
||||
"/v1/realtime/client_secrets",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=SessionClientSecretRequest(
|
||||
session=TranscriptionSession(
|
||||
audio=SessionAudio(input=SessionAudioInput(transcription=SessionTranscription(model=model)))
|
||||
)
|
||||
),
|
||||
response_type=RealtimeClientSecretResponse,
|
||||
)
|
||||
)
|
||||
assert secret.value, f"client secret value missing: {secret}"
|
||||
assert secret.session is not None and secret.session.type == "transcription", (
|
||||
f"the client secret did not mint a transcription session: {secret.session}"
|
||||
)
|
||||
audio_input: Final = secret.session.audio.input if secret.session.audio else None
|
||||
transcription: Final = audio_input.transcription if audio_input else None
|
||||
assert transcription is not None and transcription.model == backend.upstream_model, (
|
||||
f"the session does not transcribe with the deployment's model {backend.upstream_model}: {secret.session}"
|
||||
)
|
||||
|
||||
|
||||
class TestRealtimeClientSecretNulls:
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.REALTIME,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(REALTIME_BACKEND,),
|
||||
)
|
||||
)
|
||||
def test_top_level_null_is_dropped_and_nested_null_is_kept(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
secret: Final = unwrap(
|
||||
proxy.transport.post(
|
||||
"/v1/realtime/client_secrets",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=NullableClientSecretRequest(
|
||||
model=model,
|
||||
session=NullableRealtimeSession(
|
||||
type="realtime",
|
||||
model=model,
|
||||
instructions=None,
|
||||
audio=NullableAudio(input=NullableAudioInput(turn_detection=None)),
|
||||
),
|
||||
),
|
||||
response_type=RealtimeClientSecretResponse,
|
||||
)
|
||||
)
|
||||
assert secret.value, f"a session with instructions: null was not minted: {secret}"
|
||||
audio_input: Final = secret.session.audio.input if secret.session and secret.session.audio else None
|
||||
assert audio_input is not None and "turn_detection" in audio_input.model_fields_set, (
|
||||
f"the session does not report turn detection: {secret.session}"
|
||||
)
|
||||
assert audio_input.turn_detection is None, (
|
||||
f"turn_detection: null was dropped, so the session fell back to {audio_input.turn_detection}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -345,10 +345,6 @@ def test_audio_transcription_forwards_nested_and_repeated_bracket_fields_verbati
|
|||
|
||||
|
||||
def test_audio_transcription_stream_returns_server_sent_events(gateway: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: stream=true transcriptions reach the upstream but its SSE body comes back as application/json "
|
||||
'{"text": "data: ..."}, so OpenAI SDK stream=True callers get no transcript events'
|
||||
)
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
|
|
|
|||
0
tests/unit/cookbook/__init__.py
Normal file
0
tests/unit/cookbook/__init__.py
Normal file
141
tests/unit/cookbook/test_gpt_realtime_translate.py
Normal file
141
tests/unit/cookbook/test_gpt_realtime_translate.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
from cookbook import gpt_realtime_translate as translate
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.1)
|
||||
audio: Final = bytes(480)
|
||||
events: Final = iter(
|
||||
(
|
||||
{"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()},
|
||||
{"type": "error", "error": {"message": "session closed"}},
|
||||
)
|
||||
)
|
||||
|
||||
async def recv() -> str:
|
||||
event: Final = next(events)
|
||||
if event["type"] == "session.output_audio.delta":
|
||||
await asyncio.sleep(0.03)
|
||||
return json.dumps(event)
|
||||
|
||||
sender_finished: Final = asyncio.Event()
|
||||
sender_finished.set()
|
||||
output: Final = tmp_path / "translation.wav"
|
||||
|
||||
result: Final = await translate.receive_translation(
|
||||
cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished
|
||||
)
|
||||
|
||||
assert result == 'Realtime API error: {"message": "session closed"}'
|
||||
with wave.open(str(output), "rb") as rendered:
|
||||
assert rendered.readframes(240) == audio
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_upload_keeps_waiting_for_first_translated_audio_while_still_sending(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05)
|
||||
audio: Final = bytes(480)
|
||||
pending: Final[asyncio.Queue[str]] = asyncio.Queue()
|
||||
sender_finished: Final = asyncio.Event()
|
||||
|
||||
async def speak_after_a_long_pause() -> None:
|
||||
await asyncio.sleep(0.2)
|
||||
await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}))
|
||||
await asyncio.sleep(0.02)
|
||||
sender_finished.set()
|
||||
await pending.put(json.dumps({"type": "session.closed"}))
|
||||
|
||||
output: Final = tmp_path / "translation.wav"
|
||||
speaker: Final = asyncio.create_task(speak_after_a_long_pause())
|
||||
|
||||
result: Final = await translate.receive_translation(
|
||||
cast(ClientConnection, SimpleNamespace(recv=pending.get)), output, sender_finished
|
||||
)
|
||||
await speaker
|
||||
|
||||
assert result is None
|
||||
with wave.open(str(output), "rb") as rendered:
|
||||
assert rendered.readframes(240) == audio
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_quiet_gap_after_the_sender_stops_keeps_the_rest_of_the_translation(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 1.0)
|
||||
first: Final = bytes(480)
|
||||
second: Final = bytes(range(240)) * 2
|
||||
events: Final = iter(
|
||||
(
|
||||
{"type": "session.output_audio.delta", "delta": base64.b64encode(first).decode()},
|
||||
{"type": "session.output_audio.delta", "delta": base64.b64encode(second).decode()},
|
||||
{"type": "session.closed"},
|
||||
)
|
||||
)
|
||||
|
||||
async def recv() -> str:
|
||||
event: Final = next(events)
|
||||
if event.get("delta") == base64.b64encode(second).decode():
|
||||
await asyncio.sleep(0.3)
|
||||
return json.dumps(event)
|
||||
|
||||
sender_finished: Final = asyncio.Event()
|
||||
sender_finished.set()
|
||||
output: Final = tmp_path / "translation.wav"
|
||||
|
||||
result: Final = await translate.receive_translation(
|
||||
cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished
|
||||
)
|
||||
|
||||
assert result is None
|
||||
with wave.open(str(output), "rb") as rendered:
|
||||
assert rendered.readframes(480) == first + second
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stream_that_never_closes_the_session_reports_an_error(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05)
|
||||
audio: Final = bytes(480)
|
||||
pending: Final[asyncio.Queue[str]] = asyncio.Queue()
|
||||
await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}))
|
||||
sender_finished: Final = asyncio.Event()
|
||||
sender_finished.set()
|
||||
|
||||
result: Final = await translate.receive_translation(
|
||||
cast(ClientConnection, SimpleNamespace(recv=pending.get)), tmp_path / "translation.wav", sender_finished
|
||||
)
|
||||
|
||||
assert result == "The translation stream went quiet for 0.05 s without closing the session"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_sender_closes_the_session_after_the_last_audio_chunk() -> None:
|
||||
sent: Final[asyncio.Queue[str]] = asyncio.Queue()
|
||||
finished: Final = asyncio.Event()
|
||||
pcm: Final = bytes(translate.CHUNK_BYTES + 2)
|
||||
|
||||
await translate.send_audio(cast(ClientConnection, SimpleNamespace(send=sent.put)), pcm, 0.0, finished)
|
||||
|
||||
messages: Final = tuple(json.loads(sent.get_nowait()) for _ in range(sent.qsize()))
|
||||
assert [message["type"] for message in messages] == [
|
||||
"session.input_audio_buffer.append",
|
||||
"session.input_audio_buffer.append",
|
||||
"session.close",
|
||||
]
|
||||
assert b"".join(base64.b64decode(message["audio"]) for message in messages[:-1]) == pcm
|
||||
assert finished.is_set()
|
||||
|
|
@ -0,0 +1,319 @@
|
|||
import datetime
|
||||
import io
|
||||
import math
|
||||
import wave
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Final
|
||||
|
||||
import anyio
|
||||
import anyio.lowlevel
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
litellm_exception_mapper,
|
||||
wrap_transcription_stream,
|
||||
)
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
|
||||
_STARTED_AT: Final = datetime.datetime(2026, 10, 28, 12, 0, 0) # noqa: DTZ001 # callback timestamps use the legacy naive contract
|
||||
|
||||
|
||||
class _FirstDeltaThenStall(httpx.AsyncByteStream):
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
|
||||
await anyio.sleep_forever()
|
||||
|
||||
|
||||
class _FirstDeltaThenFails(httpx.AsyncByteStream):
|
||||
def __init__(self, failure: httpx.TransportError) -> None:
|
||||
self._failure = failure
|
||||
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
|
||||
raise self._failure
|
||||
|
||||
|
||||
def _transcription_router(failing_api_base: str, healthy_api_base: str) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "transcriber",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-transcribe",
|
||||
"api_key": "sk-test",
|
||||
"api_base": api_base,
|
||||
"order": order,
|
||||
},
|
||||
}
|
||||
for order, api_base in ((1, failing_api_base), (2, healthy_api_base))
|
||||
],
|
||||
allowed_fails=0,
|
||||
cooldown_time=60,
|
||||
)
|
||||
|
||||
|
||||
class _RecordingLogging:
|
||||
def __init__(self) -> None:
|
||||
self.logged: tuple[TranscriptionResponse, ...] = ()
|
||||
|
||||
def success_handler(
|
||||
self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
async def async_success_handler(
|
||||
self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime
|
||||
) -> None:
|
||||
await anyio.lowlevel.checkpoint()
|
||||
self.logged = (*self.logged, result)
|
||||
|
||||
def handle_sync_success_callbacks_for_async_calls(
|
||||
self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def failure_handler(
|
||||
self,
|
||||
exception: Exception,
|
||||
traceback_exception: str,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
*,
|
||||
for_async_request: bool = False,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
async def async_failure_handler(
|
||||
self,
|
||||
exception: Exception,
|
||||
traceback_exception: str,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stream_the_client_drops_mid_transcript_is_still_logged_as_a_success() -> None:
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenStall()
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
stream: Final = await upstream.audio.transcriptions.create(
|
||||
model="gpt-transcribe", file=("sample.wav", b"audio"), stream=True
|
||||
)
|
||||
logging_obj: Final = _RecordingLogging()
|
||||
wrapped: Final = wrap_transcription_stream(stream, logging_obj, _STARTED_AT, litellm_exception_mapper({}))
|
||||
|
||||
with anyio.CancelScope() as client_connection:
|
||||
async for _ in wrapped:
|
||||
client_connection.cancel()
|
||||
|
||||
assert [result.text for result in logging_obj.logged] == ["Good"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_deployment_whose_stream_times_out_mid_transcript_goes_into_cooldown(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
router: Final = _transcription_router("https://stalls.example/v1", "https://healthy.example/v1")
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.post("https://stalls.example/v1/audio/transcriptions").mock(
|
||||
side_effect=lambda request: httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_FirstDeltaThenFails(httpx.ReadTimeout("read timed out")),
|
||||
)
|
||||
)
|
||||
upstream.post("https://healthy.example/v1/audio/transcriptions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content=b'data: {"type":"transcript.text.done","text":"Good morning"}\n\n',
|
||||
)
|
||||
)
|
||||
stalled: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True)
|
||||
with pytest.raises(litellm.Timeout):
|
||||
async for _ in stalled:
|
||||
pass
|
||||
await anyio.sleep(0.1)
|
||||
retried: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True)
|
||||
events: Final = [event async for event in retried]
|
||||
|
||||
assert [getattr(event, "text", None) for event in events] == ["Good morning"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_stream_the_upstream_drops_mid_transcript_raises_a_litellm_error_to_the_caller() -> None:
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_FirstDeltaThenFails(httpx.RemoteProtocolError("peer closed connection")),
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError, match="peer closed connection"):
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
|
||||
class _FailureHookCounter(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.failures: tuple[str, ...] = ()
|
||||
|
||||
def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
|
||||
self.failures = (*self.failures, "sync_failure")
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: object, response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.failures = (*self.failures, "async_failure")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failure(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
counter: Final = _FailureHookCounter()
|
||||
monkeypatch.setattr(litellm, "callbacks", [counter])
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")),
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
assert counter.failures == ("async_failure",)
|
||||
|
||||
|
||||
class _FirstDeltaThenResetSync(httpx.SyncByteStream):
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
|
||||
raise httpx.ReadError("connection reset")
|
||||
|
||||
|
||||
class _FailureCostRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.costs: tuple[object, ...] = ()
|
||||
|
||||
def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
|
||||
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: object, response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
|
||||
|
||||
|
||||
def _silent_wav(seconds: int) -> bytes:
|
||||
buffer: Final = io.BytesIO()
|
||||
with wave.open(buffer, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(16_000)
|
||||
wav.writeframes(b"\x00\x00" * 16_000 * seconds)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _reset_mid_transcript(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")),
|
||||
)
|
||||
|
||||
|
||||
def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync())
|
||||
|
||||
|
||||
def _per_second_rate(model: str) -> float:
|
||||
rate: Final = litellm.model_cost[model]["input_cost_per_second"]
|
||||
assert isinstance(rate, float)
|
||||
return rate
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
recorder: Final = _FailureCostRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)),
|
||||
)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
assert len(recorder.costs) == 1
|
||||
assert isinstance(recorder.costs[0], float)
|
||||
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))
|
||||
|
||||
|
||||
def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
recorder: Final = _FailureCostRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
upstream: Final = OpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)),
|
||||
)
|
||||
stream: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
for _ in stream:
|
||||
pass
|
||||
|
||||
assert len(recorder.costs) == 1
|
||||
assert isinstance(recorder.costs[0], float)
|
||||
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))
|
||||
|
|
@ -29,6 +29,7 @@ from litellm.llms.gemini.image_generation.cost_calculator import (
|
|||
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
|
||||
cost_calculator as vertex_image_generation_cost_calculator,
|
||||
)
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.utils import (
|
||||
CacheCreationTokenDetails,
|
||||
CompletionTokensDetailsWrapper,
|
||||
|
|
@ -42,6 +43,62 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
|
||||
def test_realtime_cached_modality_breakdown_matches_prompt_cost(_local_model_cost_map):
|
||||
model: Final = "gpt-realtime-2.1-mini"
|
||||
rates: Final = litellm.model_cost[model]
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
total_tokens=1000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=400,
|
||||
audio_tokens=400,
|
||||
image_tokens=200,
|
||||
cached_tokens=300,
|
||||
cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=150, image_tokens=50),
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai")
|
||||
breakdown: Final = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage)
|
||||
cached_cost: Final = (
|
||||
100 * rates["cache_read_input_token_cost"]
|
||||
+ 150 * rates["cache_read_input_audio_token_cost"]
|
||||
+ 50 * rates["cache_read_input_image_token_cost"]
|
||||
)
|
||||
uncached_cost: Final = (
|
||||
300 * rates["input_cost_per_token"]
|
||||
+ 250 * rates["input_cost_per_audio_token"]
|
||||
+ 150 * rates["input_cost_per_image_token"]
|
||||
)
|
||||
|
||||
assert breakdown.cache_read_cost == pytest.approx(cached_cost)
|
||||
assert prompt_cost == pytest.approx(uncached_cost + cached_cost)
|
||||
|
||||
|
||||
def test_image_model_bills_cached_image_tokens_at_its_cached_image_rate(_local_model_cost_map):
|
||||
model: Final = "gpt-image-2"
|
||||
rates: Final = litellm.model_cost[model]
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
total_tokens=1000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=200,
|
||||
image_tokens=800,
|
||||
cached_tokens=600,
|
||||
cached_tokens_details=CachedTokensDetails(text_tokens=100, image_tokens=500),
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai")
|
||||
cached_cost: Final = 100 * rates["cache_read_input_token_cost"] + 500 * rates["cache_read_input_image_token_cost"]
|
||||
uncached_cost: Final = 100 * rates["input_cost_per_token"] + 300 * rates["input_cost_per_image_token"]
|
||||
|
||||
assert rates["cache_read_input_image_token_cost"] != rates["cache_read_input_token_cost"]
|
||||
assert prompt_cost == pytest.approx(uncached_cost + cached_cost)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
|
@ -2234,6 +2291,10 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
|
|||
prompt_tokens_details: PromptTokensDetailsResult = {
|
||||
"cache_hit_tokens": 0,
|
||||
"cache_hit_audio_tokens": 0,
|
||||
"cached_text_tokens": 0,
|
||||
"cached_audio_tokens": 0,
|
||||
"cached_image_tokens": 0,
|
||||
"has_cached_tokens_details": False,
|
||||
"cache_creation_tokens": 0,
|
||||
"cache_creation_token_details": CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=100,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
|
||||
|
||||
|
||||
|
|
@ -46,3 +50,18 @@ def test_langfuse_model_parameters_no_secret_leakage():
|
|||
assert "secret_fields" not in sanitized
|
||||
assert "authorization" not in sanitized
|
||||
assert "headers" not in sanitized
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hint,first_value,second_value",
|
||||
[("languages", ("en",), ("fr",)), ("keywords", ("LiteLLM",), ("Kubernetes",))],
|
||||
)
|
||||
def test_transcription_cache_key_separates_recognition_hints(
|
||||
hint: str, first_value: Sequence[str], second_value: Sequence[str]
|
||||
) -> None:
|
||||
cache: Final = Cache()
|
||||
request: Final = {"model": "gpt-4o-transcribe", "file": "meeting.wav", "response_format": "json"}
|
||||
|
||||
assert cache.get_cache_key(**request, **{hint: first_value}) != cache.get_cache_key(
|
||||
**request, **{hint: second_value}
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,7 @@
|
|||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -259,6 +262,21 @@ async def test_construct_url_v1_protocol():
|
|||
assert url.count("/realtime") == 1
|
||||
|
||||
|
||||
def test_construct_url_translation_protocol():
|
||||
from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime
|
||||
|
||||
url = AzureOpenAIRealtime()._construct_url(
|
||||
api_base="https://my-endpoint.openai.azure.com",
|
||||
model="translate-deployment",
|
||||
api_version=None,
|
||||
realtime_protocol="GA",
|
||||
query_params={"model": "translate-deployment"},
|
||||
realtime_mode="translation",
|
||||
)
|
||||
|
||||
assert url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime/translations?model=translate-deployment"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("protocol", ["ga", "Ga", "gA", "V1", "v1", "GA"])
|
||||
async def test_construct_url_case_insensitive_protocol(protocol):
|
||||
|
|
@ -752,3 +770,32 @@ def test_construct_url_with_a_trailing_slash_api_base_matches_the_url_without_it
|
|||
)
|
||||
|
||||
assert url_for("https://gw.example.com/azure/") == url_for("https://gw.example.com/azure")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ClientWebSocketWithHeaders:
|
||||
scope: Mapping[str, object]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pinned_protocol", ("GA", "ga", "v1"))
|
||||
@pytest.mark.parametrize(
|
||||
("client_headers", "bridges_beta_client_to_ga"),
|
||||
(
|
||||
pytest.param(((b"openai-beta", b"realtime=v1"),), True, id="beta-client"),
|
||||
pytest.param((), False, id="ga-client"),
|
||||
),
|
||||
)
|
||||
def test_a_deployment_pinned_to_ga_bridges_only_a_beta_client(
|
||||
pinned_protocol: str, client_headers: tuple[tuple[bytes, bytes], ...], bridges_beta_client_to_ga: bool
|
||||
) -> None:
|
||||
from litellm.llms.azure.realtime.handler import AzureRealtimeProtocol, azure_realtime_protocol_for_client
|
||||
|
||||
protocol: Final = azure_realtime_protocol_for_client(
|
||||
pinned_protocol,
|
||||
model="my-realtime-deployment",
|
||||
realtime_mode="realtime",
|
||||
query_params=None,
|
||||
websocket=_ClientWebSocketWithHeaders(scope={"headers": client_headers}),
|
||||
)
|
||||
|
||||
assert protocol == AzureRealtimeProtocol(name=pinned_protocol, bridges_beta_client_to_ga=bridges_beta_client_to_ga)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -7,7 +8,11 @@ import litellm
|
|||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
_DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview"
|
||||
_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai")
|
||||
_API_BASES: Final = (
|
||||
"https://my.openai.azure.com",
|
||||
"https://my.openai.azure.com/",
|
||||
"https://my.openai.azure.com/openai",
|
||||
)
|
||||
|
||||
|
||||
class _AzureThatRecordsTheRequest:
|
||||
|
|
@ -55,3 +60,62 @@ async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_k
|
|||
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls"
|
||||
assert request.headers.get("authorization") == "Bearer ek_upstream"
|
||||
assert "api-key" not in request.headers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "url"),
|
||||
(
|
||||
("https://gw.example.com/azure/v1", "https://gw.example.com/azure/v1/openai/realtime/transcription_sessions"),
|
||||
("https://my.openai.azure.com/openai", "https://my.openai.azure.com/openai/realtime/transcription_sessions"),
|
||||
("https://my.openai.azure.com/openai/v1", "https://my.openai.azure.com/openai/realtime/transcription_sessions"),
|
||||
),
|
||||
)
|
||||
async def test_azure_transcription_session_strips_only_a_trailing_openai_path(api_base: str, url: str) -> None:
|
||||
azure: Final = _AzureThatRecordsTheRequest()
|
||||
await litellm.acreate_realtime_transcription_session(
|
||||
model="azure/gpt-realtime-whisper-deployment",
|
||||
api_base=api_base,
|
||||
api_key="azure-key",
|
||||
api_version=_DATED_PREVIEW_API_VERSION,
|
||||
client=azure.client(),
|
||||
)
|
||||
(request,) = azure.requests
|
||||
assert str(request.url) == f"{url}?api-version={_DATED_PREVIEW_API_VERSION}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("caller_headers", "sent_headers"),
|
||||
(
|
||||
pytest.param(
|
||||
{"Authorization": "Bearer caller-token"},
|
||||
{"authorization": "Bearer caller-token", "api-key": None},
|
||||
id="caller-authorization",
|
||||
),
|
||||
pytest.param(
|
||||
{"authorization": "Bearer caller-token"},
|
||||
{"authorization": "Bearer caller-token", "api-key": None},
|
||||
id="caller-lowercase-authorization",
|
||||
),
|
||||
pytest.param({"api-key": "caller-key"}, {"authorization": None, "api-key": "caller-key"}, id="caller-api-key"),
|
||||
pytest.param({}, {"authorization": "Bearer entra-token", "api-key": None}, id="no-caller-auth"),
|
||||
),
|
||||
)
|
||||
async def test_azure_entra_fallback_never_overrides_the_callers_own_auth_header(
|
||||
caller_headers: Mapping[str, str],
|
||||
sent_headers: Mapping[str, str | None],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
no_ambient_azure_credentials: None,
|
||||
) -> None:
|
||||
monkeypatch.delenv("AZURE_API_KEY", raising=False)
|
||||
monkeypatch.setenv("AZURE_AD_TOKEN", "entra-token")
|
||||
azure: Final = _AzureThatRecordsTheRequest()
|
||||
await litellm.acreate_realtime_client_secret(
|
||||
model="azure/gpt-realtime-deployment",
|
||||
api_base="https://my.openai.azure.com",
|
||||
extra_headers=caller_headers,
|
||||
client=azure.client(),
|
||||
)
|
||||
(request,) = azure.requests
|
||||
assert {name: request.headers.get(name) for name in sent_headers} == sent_headers
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import Final
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import AsyncAzureOpenAI, AzureOpenAI
|
||||
|
||||
import litellm
|
||||
|
|
@ -160,3 +161,70 @@ async def test_azure_transcription_parses_response_formats(
|
|||
)
|
||||
|
||||
assert response.text == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("api_version", ["2024-06-01", "2025-03-01-preview", "v1", "latest", "preview"])
|
||||
async def test_azure_router_transcription_sends_the_default_api_version(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
api_version: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.delenv("AZURE_API_VERSION", raising=False)
|
||||
route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock(
|
||||
return_value=httpx.Response(200, json={"text": "hello"})
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "transcriber",
|
||||
"litellm_params": {
|
||||
"model": "azure/gpt-transcribe",
|
||||
"api_key": "test-key",
|
||||
"api_base": "https://transcribe-route.openai.azure.com",
|
||||
"api_version": api_version,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response: Final = await router.atranscription(model="transcriber", file=_audio_file())
|
||||
|
||||
sent_url: Final = route.calls.last.request.url
|
||||
assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions"
|
||||
assert sent_url.params.get("api-version") == litellm.AZURE_DEFAULT_API_VERSION
|
||||
assert response.text == "hello"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("api_version", "expected_api_version"),
|
||||
[
|
||||
("v1", litellm.AZURE_DEFAULT_API_VERSION),
|
||||
("latest", litellm.AZURE_DEFAULT_API_VERSION),
|
||||
("preview", litellm.AZURE_DEFAULT_API_VERSION),
|
||||
("2025-03-01-preview", "2025-03-01-preview"),
|
||||
],
|
||||
)
|
||||
def test_azure_sync_transcription_uses_the_deployments_route(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
api_version: str,
|
||||
expected_api_version: str,
|
||||
) -> None:
|
||||
monkeypatch.delenv("AZURE_API_VERSION", raising=False)
|
||||
route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock(
|
||||
return_value=httpx.Response(200, json={"text": "hello"})
|
||||
)
|
||||
|
||||
response: Final = litellm.transcription(
|
||||
model="azure/gpt-transcribe",
|
||||
file=_audio_file(),
|
||||
api_key="test-key",
|
||||
api_base="https://transcribe-route.openai.azure.com",
|
||||
api_version=api_version,
|
||||
)
|
||||
|
||||
sent_url: Final = route.calls.last.request.url
|
||||
assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions"
|
||||
assert sent_url.params.get("api-version") == expected_api_version
|
||||
assert response.text == "hello"
|
||||
|
|
|
|||
|
|
@ -491,6 +491,11 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
|
|||
"avector_store_search",
|
||||
"acreate_skill",
|
||||
"acreate_interaction",
|
||||
"acreate_realtime_client_secret",
|
||||
"acreate_realtime_transcription_session",
|
||||
"acreate_realtime_translation_client_secret",
|
||||
"arealtime_calls",
|
||||
"arealtime_translation_calls",
|
||||
]
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
|
@ -526,3 +530,83 @@ async def test_realtime_query_params_preserve_missing_model(monkeypatch):
|
|||
|
||||
called_kwargs = mock_async_realtime.call_args.kwargs
|
||||
assert called_kwargs["query_params"] == {"intent": "transcription"}
|
||||
|
||||
|
||||
class _BackendConnection:
|
||||
def __init__(self, connection):
|
||||
self.connection = connection
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.connection
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
|
||||
def test_translation_url_uses_dedicated_path():
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
|
||||
handler = OpenAIRealtime()
|
||||
url = handler._construct_url(
|
||||
api_base="https://api.openai.com/v1",
|
||||
query_params={"model": "gpt-realtime-translate"},
|
||||
realtime_mode="translation",
|
||||
)
|
||||
assert url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("beta_client", (False, True))
|
||||
async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> None:
|
||||
from litellm.llms.openai.realtime.handler import OpenAIRealtime
|
||||
|
||||
backend: Final = AsyncMock()
|
||||
pending: Final = asyncio.Event()
|
||||
|
||||
async def receive_backend() -> str:
|
||||
await pending.wait()
|
||||
return ""
|
||||
|
||||
backend.recv.side_effect = receive_backend
|
||||
websocket: Final = MagicMock()
|
||||
websocket.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []}
|
||||
websocket.receive_text = AsyncMock(
|
||||
side_effect=(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {"type": "translation", "audio": {"output": {"language": "fr"}}},
|
||||
}
|
||||
),
|
||||
RuntimeError("connection closed"),
|
||||
)
|
||||
)
|
||||
websocket.close = AsyncMock()
|
||||
logging_obj: Final = MagicMock(dispatch_success_handlers=AsyncMock(), dispatch_failure_handlers=AsyncMock())
|
||||
handler: Final = OpenAIRealtime()
|
||||
expected_url: Final = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate"
|
||||
assert (
|
||||
handler._construct_url(
|
||||
api_base="https://api.openai.com/v1",
|
||||
query_params={"model": "gpt-realtime-translate"},
|
||||
realtime_mode="translation",
|
||||
)
|
||||
== expected_url
|
||||
)
|
||||
|
||||
with patch("websockets.connect", return_value=_BackendConnection(backend)) as connect:
|
||||
await handler.async_realtime(
|
||||
model="gpt-realtime-translate",
|
||||
websocket=websocket,
|
||||
logging_obj=logging_obj,
|
||||
api_base="https://api.openai.com/v1",
|
||||
api_key="sk-test",
|
||||
query_params={"model": "gpt-realtime-translate"},
|
||||
realtime_mode="translation",
|
||||
)
|
||||
|
||||
connect.assert_called_once()
|
||||
assert connect.call_args.args[0] == expected_url
|
||||
assert "OpenAI-Beta" not in connect.call_args.kwargs["additional_headers"]
|
||||
sent: Final = TypeAdapter(Mapping[str, object]).validate_json(backend.send.call_args_list[0].args[0])
|
||||
assert sent["session"] == {"audio": {"output": {"language": "fr"}}}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
|
|
@ -21,9 +21,7 @@ from litellm.types.realtime import RealtimeTranscriptionSessionRequest
|
|||
def test_openai_transcription_session_url():
|
||||
cfg = OpenAIRealtimeHTTPConfig()
|
||||
assert (
|
||||
cfg.get_transcription_session_url(
|
||||
api_base="https://api.openai.com", model="gpt-realtime-whisper"
|
||||
)
|
||||
cfg.get_transcription_session_url(api_base="https://api.openai.com", model="gpt-realtime-whisper")
|
||||
== "https://api.openai.com/v1/realtime/transcription_sessions"
|
||||
)
|
||||
|
||||
|
|
@ -32,24 +30,36 @@ def test_openai_transcription_session_url_strips_trailing_v1():
|
|||
"""A /v1 suffix must not be duplicated in the path."""
|
||||
cfg = OpenAIRealtimeHTTPConfig()
|
||||
assert (
|
||||
cfg.get_transcription_session_url(
|
||||
api_base="https://api.openai.com/v1", model="gpt-realtime-whisper"
|
||||
)
|
||||
cfg.get_transcription_session_url(api_base="https://api.openai.com/v1", model="gpt-realtime-whisper")
|
||||
== "https://api.openai.com/v1/realtime/transcription_sessions"
|
||||
)
|
||||
|
||||
|
||||
def test_azure_transcription_session_url_uses_deployment_and_api_version():
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
["https://my.openai.azure.com", "https://my.openai.azure.com/openai", "https://my.openai.azure.com/openai/v1/"],
|
||||
)
|
||||
def test_azure_transcription_session_url_uses_deployment_and_api_version(api_base: str):
|
||||
cfg = AzureRealtimeHTTPConfig()
|
||||
url = cfg.get_transcription_session_url(
|
||||
api_base="https://my.openai.azure.com",
|
||||
api_base=api_base,
|
||||
model="whisper-deploy",
|
||||
api_version="2025-04-01-preview",
|
||||
)
|
||||
assert (
|
||||
url
|
||||
== "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview"
|
||||
)
|
||||
assert url == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"])
|
||||
@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"])
|
||||
def test_azure_ga_realtime_http_urls(api_version, model: str, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
cfg = AzureRealtimeHTTPConfig()
|
||||
base = "https://my.openai.azure.com"
|
||||
|
||||
assert cfg.get_complete_url(base, model, api_version) == (f"{base}/openai/v1/realtime/client_secrets")
|
||||
assert cfg.get_realtime_calls_url(base, model, api_version) == (f"{base}/openai/v1/realtime/calls")
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_request_resolves_model_returns_none_when_both_absent():
|
||||
|
|
@ -141,6 +151,27 @@ async def test_client_secret_handler_still_targets_client_secrets_url():
|
|||
assert kwargs["url"] == "https://api.openai.com/v1/realtime/client_secrets"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_client_secret_routes_on_a_provider_qualified_session_model():
|
||||
mock_response = MagicMock(spec=httpx.Response)
|
||||
mock_client = MagicMock(spec=AsyncHTTPHandler)
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
result = await litellm.acreate_realtime_client_secret(
|
||||
session={"type": "realtime", "model": "azure/gpt-realtime-2.1"},
|
||||
api_base="https://my.openai.azure.com",
|
||||
api_key="azure-test-key",
|
||||
api_version="v1",
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
assert result is mock_response
|
||||
request = mock_client.post.call_args.kwargs
|
||||
assert request["url"] == "https://my.openai.azure.com/openai/v1/realtime/client_secrets"
|
||||
assert request["headers"]["api-key"] == "azure-test-key"
|
||||
assert request["json"]["session"]["model"] == "gpt-realtime-2.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sdk_fn_routes_openai_transcription_session(monkeypatch):
|
||||
"""
|
||||
|
|
@ -169,18 +200,14 @@ async def test_sdk_fn_routes_openai_transcription_session(monkeypatch):
|
|||
assert kwargs["url"].endswith("/v1/realtime/transcription_sessions")
|
||||
# The litellm-only routing hint must not be forwarded upstream.
|
||||
assert "model" not in kwargs["json"]
|
||||
assert kwargs["json"]["input_audio_transcription"] == {
|
||||
"model": "gpt-realtime-whisper"
|
||||
}
|
||||
assert kwargs["json"]["input_audio_transcription"] == {"model": "gpt-realtime-whisper"}
|
||||
|
||||
|
||||
def test_append_query_params_skips_existing_keys():
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
|
||||
url = "wss://example.com/v1/realtime?model=gpt-4o"
|
||||
result = BaseLLMHTTPHandler._append_query_params(
|
||||
url, {"model": "ignored", "intent": "transcription"}
|
||||
)
|
||||
result = BaseLLMHTTPHandler._append_query_params(url, {"model": "ignored", "intent": "transcription"})
|
||||
assert "model=ignored" not in result
|
||||
assert "intent=transcription" in result
|
||||
|
||||
|
|
|
|||
185
tests/unit/llms/openai/realtime/test_translation.py
Normal file
185
tests/unit/llms/openai/realtime/test_translation.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig
|
||||
from litellm.types.realtime import RealtimeSessionConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
def test_azure_realtime_http_headers_use_ad_token_without_api_key():
|
||||
config = AzureRealtimeHTTPConfig()
|
||||
params = GenericLiteLLMParams(azure_ad_token="test-ad-token")
|
||||
|
||||
assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == {
|
||||
"X-Trace": "trace",
|
||||
"Authorization": "Bearer test-ad-token",
|
||||
}
|
||||
assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"}
|
||||
|
||||
|
||||
def test_realtime_session_config_supports_translation_and_live_transcription_fields():
|
||||
session = RealtimeSessionConfig(
|
||||
type="translation",
|
||||
model="gpt-realtime-translate",
|
||||
audio={
|
||||
"input": {
|
||||
"transcription": {
|
||||
"model": "gpt-live-transcribe",
|
||||
"delay": "minimal",
|
||||
"languages": ["en", "fr"],
|
||||
"keywords": ["LiteLLM"],
|
||||
}
|
||||
},
|
||||
"output": {"language": "es"},
|
||||
},
|
||||
)
|
||||
|
||||
assert session.audio is not None
|
||||
assert session.audio.input is not None
|
||||
assert session.audio.input.transcription is not None
|
||||
assert session.audio.input.transcription.delay == "minimal"
|
||||
assert session.audio.input.transcription.languages == ["en", "fr"]
|
||||
assert session.audio.input.transcription.keywords == ["LiteLLM"]
|
||||
assert session.audio.output is not None
|
||||
assert session.audio.output.language == "es"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base,expected",
|
||||
[
|
||||
(
|
||||
"https://api.openai.com",
|
||||
"https://api.openai.com/v1/realtime/translations/client_secrets",
|
||||
),
|
||||
(
|
||||
"https://api.openai.com/v1",
|
||||
"https://api.openai.com/v1/realtime/translations/client_secrets",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_openai_translation_client_secret_url(api_base: str, expected: str):
|
||||
config = OpenAIRealtimeHTTPConfig()
|
||||
assert config.get_translation_client_secret_url(api_base, "gpt-realtime-translate") == expected
|
||||
|
||||
|
||||
def test_openai_translation_calls_url():
|
||||
config = OpenAIRealtimeHTTPConfig()
|
||||
assert (
|
||||
config.get_translation_calls_url("https://api.openai.com/v1", "gpt-realtime-translate")
|
||||
== "https://api.openai.com/v1/realtime/translations/calls"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", ("", "/", "/openai", "/openai/v1/"))
|
||||
def test_azure_translation_urls_use_ga_paths(suffix: str):
|
||||
config: Final = AzureRealtimeHTTPConfig()
|
||||
api_base: Final = f"https://example.openai.azure.com{suffix}"
|
||||
assert (
|
||||
config.get_translation_client_secret_url(api_base, "translate-deployment")
|
||||
== "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets"
|
||||
)
|
||||
assert (
|
||||
config.get_translation_calls_url(api_base, "translate-deployment")
|
||||
== "https://example.openai.azure.com/openai/v1/realtime/translations/calls"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_translation_client_secret_uses_custom_translation_path():
|
||||
client = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.post = AsyncMock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"value": "ek_test"},
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/client_secrets"),
|
||||
)
|
||||
)
|
||||
logging_obj = MagicMock()
|
||||
handler = BaseLLMHTTPHandler()
|
||||
request_data = {"session": {"type": "translation", "model": "gpt-realtime-translate"}}
|
||||
|
||||
response = await handler.async_realtime_translation_client_secret_handler(
|
||||
api_base="https://api.openai.com",
|
||||
api_key="sk-test",
|
||||
request_data=request_data,
|
||||
logging_obj=logging_obj,
|
||||
timeout=10,
|
||||
provider_config=OpenAIRealtimeHTTPConfig(),
|
||||
model="gpt-realtime-translate",
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
call = client.post.call_args.kwargs
|
||||
assert call["url"] == "https://api.openai.com/v1/realtime/translations/client_secrets"
|
||||
assert call["json"] == request_data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_translation_client_secret_supports_entra_bearer_auth():
|
||||
client = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.post = AsyncMock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"value": "ek_test"},
|
||||
request=httpx.Request(
|
||||
"POST",
|
||||
"https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets",
|
||||
),
|
||||
)
|
||||
)
|
||||
handler = BaseLLMHTTPHandler()
|
||||
|
||||
await handler.async_realtime_translation_client_secret_handler(
|
||||
api_base="https://example.openai.azure.com",
|
||||
api_key="",
|
||||
request_data={"session": {"type": "translation", "model": "translate-deployment"}},
|
||||
logging_obj=MagicMock(),
|
||||
timeout=10,
|
||||
provider_config=AzureRealtimeHTTPConfig(),
|
||||
model="translate-deployment",
|
||||
extra_headers={"Authorization": "Bearer entra-token"},
|
||||
client=client,
|
||||
)
|
||||
|
||||
headers = client.post.call_args.kwargs["headers"]
|
||||
assert headers["Authorization"] == "Bearer entra-token"
|
||||
assert "api-key" not in headers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_translation_calls_use_translation_session_and_path():
|
||||
client = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.post = AsyncMock(
|
||||
return_value=httpx.Response(
|
||||
201,
|
||||
content=b"v=0\r\n",
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/calls"),
|
||||
)
|
||||
)
|
||||
logging_obj = MagicMock()
|
||||
handler = BaseLLMHTTPHandler()
|
||||
|
||||
response = await handler.async_realtime_calls_handler(
|
||||
api_base="https://api.openai.com",
|
||||
openai_ephemeral_key="ek_test",
|
||||
sdp_body=b"v=0\r\n",
|
||||
logging_obj=logging_obj,
|
||||
timeout=10,
|
||||
provider_config=OpenAIRealtimeHTTPConfig(),
|
||||
model="gpt-realtime-translate",
|
||||
client=client,
|
||||
translation=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
call = client.post.call_args.kwargs
|
||||
assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls"
|
||||
assert call["headers"]["Content-Type"] == "application/sdp"
|
||||
assert call["content"] == "v=0\r\n"
|
||||
868
tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py
Normal file
868
tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py
Normal file
|
|
@ -0,0 +1,868 @@
|
|||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import queue
|
||||
import wave
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import datetime
|
||||
from typing import Final, NamedTuple
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
|
||||
litellm_exception_mapper,
|
||||
wrap_transcription_stream,
|
||||
)
|
||||
from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.openai.transcriptions.gpt_transformation import (
|
||||
OpenAIGPTTranscribeAudioTranscriptionConfig,
|
||||
)
|
||||
from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
from litellm.utils import get_optional_params_transcription
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_gpt_transcribe_config_uses_native_parameters_and_json():
|
||||
config = OpenAIGPTTranscribeAudioTranscriptionConfig()
|
||||
supported = config.get_supported_openai_params("gpt-transcribe")
|
||||
assert supported == ["prompt", "response_format", "keywords", "languages", "stream"]
|
||||
|
||||
audio_file = io.BytesIO(b"audio")
|
||||
request = config.transform_audio_transcription_request(
|
||||
model="gpt-transcribe",
|
||||
audio_file=audio_file,
|
||||
optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True, "response_format": "json"},
|
||||
litellm_params={},
|
||||
)
|
||||
assert request.data["response_format"] == "json"
|
||||
assert request.data["keywords"] == ["LiteLLM"]
|
||||
assert request.data["languages"] == ["en", "fr"]
|
||||
assert request.data["stream"] is True
|
||||
|
||||
|
||||
def test_gpt_transcribe_optional_params_are_preserved():
|
||||
params = get_optional_params_transcription(
|
||||
model="gpt-transcribe",
|
||||
custom_llm_provider="openai",
|
||||
keywords=["LiteLLM", "Realtime API"],
|
||||
languages=["en", "fr"],
|
||||
stream=True,
|
||||
)
|
||||
assert params == {
|
||||
"keywords": ["LiteLLM", "Realtime API"],
|
||||
"languages": ["en", "fr"],
|
||||
"stream": True,
|
||||
}
|
||||
|
||||
|
||||
def test_transcription_response_preserves_empty_languages():
|
||||
response = TranscriptionResponse(text="hello", languages=[])
|
||||
assert json.loads(response.model_dump_json())["languages"] == []
|
||||
|
||||
|
||||
def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None:
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content=(
|
||||
'data: {"type":"transcript.text.delta","delta":"hello "}\n\n'
|
||||
'data: {"type":"transcript.text.done","text":"hello world",'
|
||||
'"usage":{"type":"duration","seconds":2.5},"languages":[{"code":"en"}]}\n\n'
|
||||
),
|
||||
)
|
||||
|
||||
logging_obj: Final = MagicMock()
|
||||
with OpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(send_response)),
|
||||
) as client:
|
||||
stream: Final = client.audio.transcriptions.create(
|
||||
model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True
|
||||
)
|
||||
wrapped: Final = wrap_transcription_stream(stream, logging_obj, datetime(2026, 1, 1), litellm_exception_mapper({}))
|
||||
received: Final = tuple(wrapped)
|
||||
wrapped.close()
|
||||
|
||||
assert tuple(event.type for event in received) == ("transcript.text.delta", "transcript.text.done")
|
||||
logging_obj.success_handler.assert_called_once()
|
||||
logged_response: Final = logging_obj.success_handler.call_args.args[0]
|
||||
assert logged_response.text == "hello world"
|
||||
assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5}
|
||||
assert json.loads(logged_response.model_dump_json())["languages"] == [{"code": "en"}]
|
||||
logging_obj.failure_handler.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_handler_returns_native_typed_stream():
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
body = await request.aread()
|
||||
assert b'name="keywords[]"' in body
|
||||
assert b'name="languages[]"' in body
|
||||
assert b'name="stream"' in body
|
||||
events = (
|
||||
{"type": "transcript.text.delta", "delta": "hello "},
|
||||
{
|
||||
"type": "transcript.text.done",
|
||||
"text": "hello world",
|
||||
"languages": [],
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 2,
|
||||
"total_tokens": 12,
|
||||
},
|
||||
},
|
||||
)
|
||||
content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode()
|
||||
return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"})
|
||||
|
||||
http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response))
|
||||
openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client)
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.model_call_details = {}
|
||||
audio_file = io.BytesIO(b"audio")
|
||||
audio_file.name = "sample.wav"
|
||||
|
||||
handler = OpenAIAudioTranscription()
|
||||
result = handler.audio_transcriptions(
|
||||
model="gpt-transcribe",
|
||||
audio_file=audio_file,
|
||||
optional_params={"keywords": ["LiteLLM"], "languages": ["en"], "stream": True},
|
||||
litellm_params={},
|
||||
model_response=TranscriptionResponse(),
|
||||
timeout=10,
|
||||
max_retries=0,
|
||||
logging_obj=logging_obj,
|
||||
api_key="sk-test",
|
||||
api_base="https://example.com/v1",
|
||||
client=openai_client,
|
||||
atranscription=True,
|
||||
provider_config=OpenAIGPTTranscribeAudioTranscriptionConfig(),
|
||||
)
|
||||
stream = await result
|
||||
assert isinstance(stream, AsyncStream)
|
||||
logging_obj.async_success_handler = AsyncMock()
|
||||
logging_obj.async_failure_handler = AsyncMock()
|
||||
wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now(), litellm_exception_mapper({}))
|
||||
received = [event async for event in wrapped_stream]
|
||||
await wrapped_stream.close()
|
||||
await openai_client.close()
|
||||
|
||||
assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"]
|
||||
assert received[-1].languages == []
|
||||
logging_obj.async_success_handler.assert_awaited_once()
|
||||
logged_response = logging_obj.async_success_handler.await_args.kwargs["result"]
|
||||
assert logged_response.text == "hello world"
|
||||
assert logged_response.languages == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_atranscription_stream_preserves_duration_for_callback_cost():
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
events = (
|
||||
{"type": "transcript.text.delta", "delta": "hello "},
|
||||
{
|
||||
"type": "transcript.text.done",
|
||||
"text": "hello world",
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 2,
|
||||
"total_tokens": 12,
|
||||
},
|
||||
},
|
||||
)
|
||||
content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode()
|
||||
return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"})
|
||||
|
||||
http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response))
|
||||
openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
logging_obj = Logging(
|
||||
model="gpt-transcribe", messages=[], stream=True, call_type="atranscription",
|
||||
start_time=datetime.now(), litellm_call_id="transcription-cost-test", function_id="transcription-cost-test",
|
||||
)
|
||||
audio_file = io.BytesIO()
|
||||
with wave.open(audio_file, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(16000)
|
||||
wav_file.writeframes(b"\x00\x00" * 16000)
|
||||
audio_file.name = "sample.wav"
|
||||
|
||||
stream = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe",
|
||||
file=audio_file,
|
||||
stream=True,
|
||||
client=openai_client,
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
received = [event async for event in stream]
|
||||
await stream.close()
|
||||
await openai_client.close()
|
||||
|
||||
assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"]
|
||||
logged_response = logging_obj.model_call_details["async_complete_streaming_response"]
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.000075)
|
||||
assert logging_obj.model_call_details["standard_logging_object"]["response_cost"] == pytest.approx(0.000075)
|
||||
assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"])
|
||||
def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None:
|
||||
audio_file = io.BytesIO(b"audio")
|
||||
audio_file.name = "sample.wav"
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"):
|
||||
litellm.transcription(
|
||||
model=model,
|
||||
file=audio_file,
|
||||
response_format="verbose_json",
|
||||
api_key="sk-test",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_globally", (True, False))
|
||||
def test_gpt_transcribe_drops_whisper_response_formats_under_drop_params(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, drop_globally: bool
|
||||
) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}})
|
||||
|
||||
monkeypatch.setattr(litellm, "drop_params", drop_globally)
|
||||
audio_file: Final = io.BytesIO(b"audio")
|
||||
audio_file.name = "sample.wav"
|
||||
with OpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(send_response)),
|
||||
) as client:
|
||||
response: Final = litellm.transcription(
|
||||
model="gpt-transcribe",
|
||||
file=audio_file,
|
||||
response_format="verbose_json",
|
||||
client=client,
|
||||
**({} if drop_globally else {"drop_params": True}),
|
||||
)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert len(sent_bodies) == 1
|
||||
assert b"verbose_json" not in sent_bodies[0]
|
||||
|
||||
|
||||
def _openai_transcription_client(transport: httpx.MockTransport) -> OpenAI:
|
||||
return OpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=httpx.Client(transport=transport))
|
||||
|
||||
|
||||
def _azure_transcription_client(transport: httpx.MockTransport) -> AzureOpenAI:
|
||||
return AzureOpenAI(
|
||||
api_key="azure-test-key",
|
||||
azure_endpoint="https://example.openai.azure.com",
|
||||
api_version=litellm.AZURE_DEFAULT_API_VERSION,
|
||||
http_client=httpx.Client(transport=transport),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "make_client"),
|
||||
(("gpt-transcribe", _openai_transcription_client), ("azure/gpt-transcribe", _azure_transcription_client)),
|
||||
)
|
||||
def test_gpt_transcribe_sends_text_response_format_to_the_provider(
|
||||
local_model_cost_map: None, model: str, make_client: Callable[[httpx.MockTransport], OpenAI | AzureOpenAI]
|
||||
) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return httpx.Response(200, text="hello", headers={"content-type": "text/plain"})
|
||||
|
||||
audio_file: Final = io.BytesIO(b"audio")
|
||||
audio_file.name = "sample.wav"
|
||||
with make_client(httpx.MockTransport(send_response)) as client:
|
||||
response: Final = litellm.transcription(
|
||||
model=model,
|
||||
file=audio_file,
|
||||
response_format="text",
|
||||
api_key="sk-test",
|
||||
api_base="https://example.openai.azure.com",
|
||||
api_version=litellm.AZURE_DEFAULT_API_VERSION,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert len(sent_bodies) == 1
|
||||
assert b'name="response_format"\r\n\r\ntext\r\n' in sent_bodies[0]
|
||||
|
||||
|
||||
def test_file_transcription_reaches_models_whose_cost_map_lists_other_endpoints(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
sent_requests: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock upstreams received
|
||||
|
||||
def send_gemini_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "interaction-1",
|
||||
"status": "completed",
|
||||
"steps": [
|
||||
{"type": "model_generation", "content": [{"type": "text", "text": "hello", "annotations": []}]}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def send_whisper_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return httpx.Response(200, json={"text": "hello"})
|
||||
|
||||
gemini_response: Final = litellm.transcription(
|
||||
model="gemini/gemini-2.5-flash",
|
||||
file=_one_second_wav(),
|
||||
api_key="gemini-test-key",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send_gemini_response))),
|
||||
)
|
||||
monkeypatch.setattr(litellm, "client_session", httpx.Client(transport=httpx.MockTransport(send_whisper_response)))
|
||||
whisper_response: Final = litellm.transcription(
|
||||
model="openai/gpt-realtime-whisper",
|
||||
file=_one_second_wav(),
|
||||
api_key="self-hosted-whisper-key",
|
||||
api_base="https://whisper.self-hosted.example/v1",
|
||||
)
|
||||
|
||||
assert gemini_response.text == "hello"
|
||||
assert whisper_response.text == "hello"
|
||||
assert tuple(str(request.url.host) for request in sent_requests) == (
|
||||
"generativelanguage.googleapis.com",
|
||||
"whisper.self-hosted.example",
|
||||
)
|
||||
|
||||
|
||||
def test_azure_gpt_transcribe_uses_deployment_scoped_route():
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
assert str(request.url) == (
|
||||
"https://example.openai.azure.com/openai/deployments/gpt-transcribe/audio/transcriptions"
|
||||
f"?api-version={litellm.AZURE_DEFAULT_API_VERSION}"
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"text": "hello", "languages": [{"code": "en"}], "usage": {"type": "duration", "seconds": 1}},
|
||||
)
|
||||
|
||||
http_client = httpx.Client(transport=httpx.MockTransport(send_response))
|
||||
client = AzureOpenAI(
|
||||
api_key="azure-test-key",
|
||||
azure_endpoint="https://example.openai.azure.com",
|
||||
api_version=litellm.AZURE_DEFAULT_API_VERSION,
|
||||
http_client=http_client,
|
||||
)
|
||||
audio_file = io.BytesIO(b"audio")
|
||||
audio_file.name = "sample.wav"
|
||||
|
||||
response = AzureAudioTranscription().audio_transcriptions(
|
||||
model="gpt-transcribe",
|
||||
audio_file=audio_file,
|
||||
optional_params={"response_format": "json"},
|
||||
logging_obj=MagicMock(),
|
||||
model_response=TranscriptionResponse(),
|
||||
timeout=10,
|
||||
max_retries=0,
|
||||
api_key="azure-test-key",
|
||||
api_base="https://example.openai.azure.com",
|
||||
api_version=litellm.AZURE_DEFAULT_API_VERSION,
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert response.languages is not None
|
||||
assert [language.code for language in response.languages] == ["en"]
|
||||
client.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body():
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
body = await request.aread()
|
||||
assert b'name="keywords[]"' in body
|
||||
assert b'name="languages[]"' in body
|
||||
return httpx.Response(200, json={"text": "hello"})
|
||||
|
||||
http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response))
|
||||
client = AsyncAzureOpenAI(
|
||||
api_key="azure-test-key",
|
||||
azure_endpoint="https://example.openai.azure.com",
|
||||
api_version="2025-04-01-preview",
|
||||
http_client=http_client,
|
||||
)
|
||||
audio_file = io.BytesIO(b"audio")
|
||||
audio_file.name = "sample.wav"
|
||||
|
||||
response = await AzureAudioTranscription().audio_transcriptions(
|
||||
model="gpt-transcribe",
|
||||
audio_file=audio_file,
|
||||
optional_params={"keywords": ["LiteLLM"], "languages": ["en"]},
|
||||
logging_obj=MagicMock(),
|
||||
model_response=TranscriptionResponse(),
|
||||
timeout=10,
|
||||
max_retries=0,
|
||||
api_key="azure-test-key",
|
||||
api_base="https://example.openai.azure.com",
|
||||
api_version="2025-04-01-preview",
|
||||
client=client,
|
||||
atranscription=True,
|
||||
)
|
||||
|
||||
assert response.text == "hello"
|
||||
await client.close()
|
||||
|
||||
|
||||
def _one_second_wav() -> io.BytesIO:
|
||||
audio_file: Final = io.BytesIO()
|
||||
with wave.open(audio_file, "wb") as wav_file:
|
||||
wav_file.setnchannels(1)
|
||||
wav_file.setsampwidth(2)
|
||||
wav_file.setframerate(16000)
|
||||
wav_file.writeframes(b"\x00\x00" * 16000)
|
||||
audio_file.name = "sample.wav"
|
||||
audio_file.seek(0)
|
||||
return audio_file
|
||||
|
||||
|
||||
class _LoggedSpend(NamedTuple):
|
||||
cost: float | None
|
||||
usage: dict[str, object] | None
|
||||
|
||||
|
||||
def _logged_spend(kwargs: dict[str, object]) -> _LoggedSpend:
|
||||
payload: Final = kwargs["standard_logging_object"]
|
||||
return _LoggedSpend(payload["response_cost"], payload["metadata"]["usage_object"])
|
||||
|
||||
|
||||
class _SuccessSpendRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.async_spends: Final[asyncio.Queue[_LoggedSpend]] = asyncio.Queue()
|
||||
self.sync_spends: Final[queue.Queue[_LoggedSpend]] = queue.Queue()
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
await self.async_spends.put(_logged_spend(kwargs))
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
self.sync_spends.put(_logged_spend(kwargs))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_atranscription_bills_the_transcript_when_the_model_ignores_stream(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
return _duration_priced_transcript(streamed=False)
|
||||
|
||||
recorder: Final = _SuccessSpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
client: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)),
|
||||
)
|
||||
await litellm.atranscription(model="openai/whisper-1", file=_one_second_wav(), client=client)
|
||||
unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
ignored_stream: Final = await litellm.atranscription(
|
||||
model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client
|
||||
)
|
||||
ignored_stream_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
await client.close()
|
||||
|
||||
assert isinstance(ignored_stream, TranscriptionResponse)
|
||||
assert ignored_stream.text == "hello world"
|
||||
assert unstreamed_spend.cost is not None
|
||||
assert unstreamed_spend.cost > 0
|
||||
assert unstreamed_spend.usage
|
||||
assert ignored_stream_spend == unstreamed_spend
|
||||
|
||||
|
||||
def test_azure_transcription_bills_the_transcript_when_the_deployment_ignores_stream(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
return _duration_priced_transcript(streamed=False)
|
||||
|
||||
recorder: Final = _SuccessSpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
client: Final = AzureOpenAI(
|
||||
api_key="azure-test-key",
|
||||
azure_endpoint="https://example.openai.azure.com",
|
||||
api_version=litellm.AZURE_DEFAULT_API_VERSION,
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(send_response)),
|
||||
)
|
||||
azure_params: Final = {
|
||||
"client": client,
|
||||
"api_key": "azure-test-key",
|
||||
"api_base": "https://example.openai.azure.com",
|
||||
"api_version": litellm.AZURE_DEFAULT_API_VERSION,
|
||||
}
|
||||
litellm.transcription(model="azure/whisper-1", file=_one_second_wav(), **azure_params)
|
||||
unstreamed_spend: Final = recorder.sync_spends.get(timeout=5)
|
||||
ignored_stream: Final = litellm.transcription(
|
||||
model="azure/whisper-1", file=_one_second_wav(), stream=True, **azure_params
|
||||
)
|
||||
ignored_stream_spend: Final = recorder.sync_spends.get(timeout=5)
|
||||
client.close()
|
||||
|
||||
assert isinstance(ignored_stream, TranscriptionResponse)
|
||||
assert ignored_stream.text == "hello world"
|
||||
assert unstreamed_spend.cost is not None
|
||||
assert unstreamed_spend.cost > 0
|
||||
assert unstreamed_spend.usage
|
||||
assert ignored_stream_spend == unstreamed_spend
|
||||
|
||||
|
||||
def _duration_priced_transcript(streamed: bool) -> httpx.Response:
|
||||
usage: Final = {"type": "duration", "seconds": 2}
|
||||
if not streamed:
|
||||
return httpx.Response(200, json={"text": "hello world", "usage": usage})
|
||||
events: Final = (
|
||||
{"type": "transcript.text.delta", "delta": "hello world"},
|
||||
{"type": "transcript.text.done", "text": "hello world", "usage": usage},
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
content="".join(f"data: {json.dumps(event)}\n\n" for event in events).encode(),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
|
||||
def test_whisper_stream_request_answered_with_json_keeps_the_unstreamed_response_shape(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
verbose_transcript: Final = {
|
||||
"task": "transcribe",
|
||||
"language": "english",
|
||||
"duration": 1.0,
|
||||
"text": "hello world",
|
||||
"usage": {"type": "duration", "seconds": 1},
|
||||
}
|
||||
with _openai_transcription_client(
|
||||
httpx.MockTransport(lambda request: httpx.Response(200, json=verbose_transcript))
|
||||
) as client:
|
||||
unstreamed: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client)
|
||||
ignored_stream: Final = litellm.transcription(
|
||||
model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client
|
||||
)
|
||||
|
||||
assert isinstance(ignored_stream, TranscriptionResponse)
|
||||
assert json.loads(ignored_stream.model_dump_json()) == json.loads(unstreamed.model_dump_json())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested_stream", ("true", "True", "1", "yes", "on"))
|
||||
def test_transcription_stream_requested_as_a_form_string_streams_like_a_bool(
|
||||
local_model_cost_map: None, requested_stream: str
|
||||
) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return _duration_priced_transcript(streamed=_asks_for_stream(sent_bodies[-1]))
|
||||
|
||||
with _openai_transcription_client(httpx.MockTransport(send_response)) as client:
|
||||
stream: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=_one_second_wav(), stream=requested_stream, client=client
|
||||
)
|
||||
events: Final = [event.type for event in stream]
|
||||
|
||||
assert events == ["transcript.text.delta", "transcript.text.done"]
|
||||
assert len(sent_bodies) == 1
|
||||
|
||||
|
||||
def test_transcription_stream_that_is_not_a_boolean_answers_400_without_calling_the_upstream(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return _duration_priced_transcript(streamed=False)
|
||||
|
||||
with _openai_transcription_client(httpx.MockTransport(send_response)) as client:
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), stream="maybe", client=client)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert sent_bodies == []
|
||||
|
||||
|
||||
def test_transcription_with_an_empty_stream_form_value_runs_unstreamed_without_a_stream_field(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return _duration_priced_transcript(streamed=False)
|
||||
|
||||
with _openai_transcription_client(httpx.MockTransport(send_response)) as client:
|
||||
response: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=_one_second_wav(), stream="", client=client
|
||||
)
|
||||
|
||||
assert isinstance(response, TranscriptionResponse)
|
||||
assert response.text == "hello world"
|
||||
assert len(sent_bodies) == 1
|
||||
assert b'name="stream"' not in sent_bodies[0]
|
||||
|
||||
|
||||
def test_transcription_stream_that_is_not_a_boolean_is_dropped_under_drop_params(local_model_cost_map: None) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return _duration_priced_transcript(streamed=False)
|
||||
|
||||
with _openai_transcription_client(httpx.MockTransport(send_response)) as client:
|
||||
response: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=_one_second_wav(), stream="maybe", drop_params=True, client=client
|
||||
)
|
||||
|
||||
assert isinstance(response, TranscriptionResponse)
|
||||
assert response.text == "hello world"
|
||||
assert len(sent_bodies) == 1
|
||||
assert b'name="stream"' not in sent_bodies[0]
|
||||
|
||||
|
||||
def _asks_for_stream(request_body: bytes) -> bool:
|
||||
return b'name="stream"\r\n\r\ntrue' in request_body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_atranscription_stream_bills_the_provider_duration_like_the_unstreamed_call(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread()))
|
||||
|
||||
recorder: Final = _SuccessSpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
client: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)),
|
||||
)
|
||||
await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client)
|
||||
unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client
|
||||
)
|
||||
events: Final = [event.type async for event in stream]
|
||||
streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
await client.close()
|
||||
|
||||
assert events == ["transcript.text.delta", "transcript.text.done"]
|
||||
assert unstreamed_spend.cost is not None
|
||||
assert unstreamed_spend.cost > 0
|
||||
assert unstreamed_spend.usage
|
||||
assert streamed_spend == unstreamed_spend
|
||||
|
||||
|
||||
def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
return _duration_priced_transcript(streamed=_asks_for_stream(request.read()))
|
||||
|
||||
recorder: Final = _SuccessSpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
client: Final = OpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(send_response)),
|
||||
)
|
||||
litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client)
|
||||
unstreamed_spend: Final = recorder.sync_spends.get(timeout=5)
|
||||
stream: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client
|
||||
)
|
||||
events: Final = [event.type for event in stream]
|
||||
streamed_spend: Final = recorder.sync_spends.get(timeout=5)
|
||||
client.close()
|
||||
|
||||
assert events == ["transcript.text.delta", "transcript.text.done"]
|
||||
assert unstreamed_spend.cost is not None
|
||||
assert unstreamed_spend.cost > 0
|
||||
assert unstreamed_spend.usage
|
||||
assert streamed_spend == unstreamed_spend
|
||||
|
||||
|
||||
class _LoggedResponseRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.responses: Final[queue.Queue[object]] = queue.Queue()
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
self.responses.put(kwargs["standard_logging_object"]["response"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_body",
|
||||
({"text": "hello"}, {"text": "hello", "languages": ["en"]}, {"text": "hello", "languages": None}),
|
||||
)
|
||||
def test_transcription_without_parsed_languages_has_no_languages_key(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object]
|
||||
) -> None:
|
||||
recorder: Final = _LoggedResponseRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
with _openai_transcription_client(
|
||||
httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body))
|
||||
) as client:
|
||||
response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client)
|
||||
logged_response: Final = recorder.responses.get(timeout=5)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert "languages" not in json.loads(response.model_dump_json())
|
||||
assert isinstance(logged_response, dict)
|
||||
assert "languages" not in logged_response
|
||||
|
||||
|
||||
def test_unstreamed_transcription_sends_no_stream_field(local_model_cost_map: None) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return httpx.Response(200, json={"text": "hello"})
|
||||
|
||||
with _openai_transcription_client(httpx.MockTransport(send_response)) as client:
|
||||
response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert len(sent_bodies) == 1
|
||||
assert b'name="stream"' not in sent_bodies[0]
|
||||
|
||||
|
||||
def test_transcription_forwards_both_language_and_languages(local_model_cost_map: None) -> None:
|
||||
sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received
|
||||
|
||||
def send_response(request: httpx.Request) -> httpx.Response:
|
||||
sent_bodies.append(request.read())
|
||||
return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}})
|
||||
|
||||
with _openai_transcription_client(httpx.MockTransport(send_response)) as client:
|
||||
response: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=_one_second_wav(), language="en", languages=["fr"], client=client
|
||||
)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert len(sent_bodies) == 1
|
||||
assert b'name="language"\r\n\r\nen\r\n' in sent_bodies[0]
|
||||
assert b'name="languages[]"\r\n\r\nfr\r\n' in sent_bodies[0]
|
||||
|
||||
|
||||
def _gpt_transcribe_cost_per_second() -> float:
|
||||
rate: Final = litellm.model_cost["gpt-transcribe"]["input_cost_per_second"]
|
||||
assert isinstance(rate, float)
|
||||
return rate
|
||||
|
||||
|
||||
async def _spend_for_one_second_wav(
|
||||
monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object], router: litellm.Router | None
|
||||
) -> _LoggedSpend:
|
||||
recorder: Final = _SuccessSpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
client: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body))
|
||||
),
|
||||
)
|
||||
if router is None:
|
||||
await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client)
|
||||
else:
|
||||
await router.atranscription(model="my-transcribe", file=_one_second_wav(), client=client)
|
||||
spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
await client.close()
|
||||
return spend
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_named_gpt_transcribe_deployment_bills_the_measured_upload_over_reported_seconds(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-transcribe",
|
||||
"litellm_params": {"model": "openai/my-transcribe", "api_key": "sk-test"},
|
||||
"model_info": {"base_model": "gpt-transcribe"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
spend: Final = await _spend_for_one_second_wav(
|
||||
monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 2}}, router
|
||||
)
|
||||
|
||||
assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gpt_transcribe_bills_the_file_length_when_the_provider_reports_zero_seconds(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
spend: Final = await _spend_for_one_second_wav(
|
||||
monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 0}}, None
|
||||
)
|
||||
|
||||
assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_m4a_transcription_stream_that_ends_early_logs_one_spend_row_like_the_unstreamed_call(
|
||||
local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
async def send_response(request: httpx.Request) -> httpx.Response:
|
||||
if not _asks_for_stream(await request.aread()):
|
||||
return httpx.Response(200, json={"text": "hello"})
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=f"data: {json.dumps({'type': 'transcript.text.delta', 'delta': 'hello'})}\n\n".encode(),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
recorder: Final = _SuccessSpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
client: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://example.com/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)),
|
||||
)
|
||||
await litellm.atranscription(model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), client=client)
|
||||
unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), stream=True, client=client
|
||||
)
|
||||
events: Final = [event.type async for event in stream]
|
||||
streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5)
|
||||
await asyncio.sleep(0.2)
|
||||
await client.close()
|
||||
|
||||
assert events == ["transcript.text.delta"]
|
||||
assert streamed_spend.cost == unstreamed_spend.cost
|
||||
assert recorder.async_spends.empty()
|
||||
|
|
@ -9,6 +9,9 @@ TranscriptionVerbose/Diarized type.
|
|||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.cost_calculator import completion_cost
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
convert_to_model_response_object,
|
||||
|
|
@ -55,10 +58,7 @@ class TestDiarizedJsonUsageParsing:
|
|||
assert result.usage.seconds == 295.8
|
||||
|
||||
def test_usage_duration_object_accepts_float_seconds(self):
|
||||
assert (
|
||||
TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds
|
||||
== 295.8
|
||||
)
|
||||
assert TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds == 295.8
|
||||
|
||||
|
||||
class TestTranscriptionDurationNotInResponseBody:
|
||||
|
|
@ -126,6 +126,61 @@ class TestTranscriptionDurationNotInResponseBody:
|
|||
class TestCostCalculatorReadsDurationFromHiddenParams:
|
||||
"""The cost calculator should read duration from _hidden_params via completion_cost()."""
|
||||
|
||||
@patch( # test-quality-ok: isolates duration selection from the model pricing table
|
||||
"litellm.cost_calculator.openai_cost_per_second"
|
||||
)
|
||||
def test_completion_cost_prefers_measured_duration_over_provider_usage_duration(self, mock_cost_fn):
|
||||
mock_cost_fn.return_value = (0.001, 0.0)
|
||||
response = TranscriptionResponse(
|
||||
text="test",
|
||||
usage={"type": "duration", "seconds": 60.0},
|
||||
)
|
||||
response._hidden_params = {
|
||||
"audio_transcription_duration": 17.5,
|
||||
"custom_llm_provider": "openai",
|
||||
}
|
||||
|
||||
completion_cost(
|
||||
completion_response=response,
|
||||
model="gpt-transcribe",
|
||||
call_type="atranscription",
|
||||
)
|
||||
|
||||
mock_cost_fn.assert_called_once()
|
||||
_, kwargs = mock_cost_fn.call_args
|
||||
assert kwargs["duration"] == 17.5
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "provider"),
|
||||
[
|
||||
("gpt-transcribe", "openai"),
|
||||
("azure/gpt-transcribe", "azure"),
|
||||
],
|
||||
)
|
||||
def test_gpt_transcribe_provider_usage_duration_costs_without_local_decode(
|
||||
self,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
provider: str,
|
||||
):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
monkeypatch.setitem(litellm.model_cost, model, {**litellm.model_cost[model], "input_cost_per_second": 0.0125})
|
||||
response = TranscriptionResponse(
|
||||
text="test",
|
||||
usage={"type": "duration", "seconds": 60.0},
|
||||
)
|
||||
response._hidden_params = {"custom_llm_provider": provider}
|
||||
|
||||
actual = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
call_type="atranscription",
|
||||
custom_llm_provider=provider,
|
||||
)
|
||||
|
||||
assert actual == pytest.approx(60.0 * 0.0125)
|
||||
|
||||
@patch("litellm.cost_calculator.openai_cost_per_second")
|
||||
def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -364,6 +364,10 @@ async def test_unknown_invocation_target_leaves_billing_unset(monkeypatch: pytes
|
|||
("/v1beta/models/gemini-model:generateContent", "POST", True),
|
||||
("/v1/realtime", "GET", True),
|
||||
("/v1/realtime", "POST", False),
|
||||
("/v1/realtime/translations", "GET", True),
|
||||
("/v1/realtime/translations", "POST", False),
|
||||
("/v1/realtime/translations/client_secrets", "POST", False),
|
||||
("/v1/realtime/translations/calls", "POST", False),
|
||||
("/v1/realtime/client_secrets", "POST", False),
|
||||
("/mcp/tools/call", "POST", True),
|
||||
("/a2a/target/message/send", "POST", True),
|
||||
|
|
@ -437,14 +441,21 @@ def test_managed_inference_ignores_unsupported_query_model():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["/realtime", "/v1/realtime", "/openai/v1/realtime"])
|
||||
@pytest.mark.parametrize(
|
||||
"route",
|
||||
(
|
||||
"/realtime",
|
||||
"/v1/realtime",
|
||||
"/openai/v1/realtime",
|
||||
),
|
||||
)
|
||||
def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route: str) -> None:
|
||||
from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request
|
||||
|
||||
with pytest.raises(HTTPException, match="explicit or configured model"):
|
||||
managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli")
|
||||
assert (
|
||||
managed_inference_request(route, {"model": "requested"}, {"completion_model": "allowed-default"}, "cli")[
|
||||
managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli", query_model="requested")[
|
||||
"model"
|
||||
]
|
||||
== "requested"
|
||||
|
|
@ -465,6 +476,20 @@ def test_managed_live_upgrade_defers_the_model_to_the_session_start_frame(route:
|
|||
) == {"model": "frame-model"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/realtime/translations", "/v1/realtime/translations"))
|
||||
@pytest.mark.parametrize("query_model", (None, "requested"))
|
||||
def test_managed_translation_authorizes_the_same_model_as_the_endpoint(route: str, query_model: str | None) -> None:
|
||||
from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request
|
||||
from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model
|
||||
|
||||
admitted: Final = managed_inference_request(
|
||||
route, {}, {"completion_model": "unrelated-default"}, "cli", query_model=query_model
|
||||
)
|
||||
|
||||
assert admitted["model"] == resolve_realtime_route_model(query_model, None, is_translation=True)
|
||||
assert isinstance(admitted["model"], str) and admitted["model"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode,user", [("autonomous", None), ("delegated", "verified-human")])
|
||||
def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, user: str | None) -> None:
|
||||
context: Final = ManagedAgentContext.model_validate(
|
||||
|
|
@ -573,21 +598,41 @@ async def test_ordinary_agent_admission_preserves_legacy_authentication(
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"route",
|
||||
tuple(dict.fromkeys(
|
||||
LiteLLMRoutes.openai_routes.value
|
||||
+ LiteLLMRoutes.anthropic_routes.value
|
||||
+ LiteLLMRoutes.google_routes.value
|
||||
)),
|
||||
tuple(
|
||||
dict.fromkeys(
|
||||
LiteLLMRoutes.openai_routes.value + LiteLLMRoutes.anthropic_routes.value + LiteLLMRoutes.google_routes.value
|
||||
)
|
||||
),
|
||||
)
|
||||
def test_registered_inference_routes_have_an_explicit_managed_access_decision(route: str) -> None:
|
||||
from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed
|
||||
|
||||
normalized: Final = route.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
|
||||
unsupported: Final = normalized.startswith((
|
||||
"/videos", "/batches", "/files", "/fine_tuning", "/assistants", "/threads", "/utils/",
|
||||
"/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/",
|
||||
"/interactions", "/agents", "/responses/{", "/responses/input_tokens",
|
||||
"/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions",
|
||||
)) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models")
|
||||
unsupported: Final = normalized.startswith(
|
||||
(
|
||||
"/videos",
|
||||
"/batches",
|
||||
"/files",
|
||||
"/fine_tuning",
|
||||
"/assistants",
|
||||
"/threads",
|
||||
"/utils/",
|
||||
"/vector_stores",
|
||||
"/vector_store/",
|
||||
"/search",
|
||||
"/containers",
|
||||
"/skills",
|
||||
"/claude-code/",
|
||||
"/interactions",
|
||||
"/agents",
|
||||
"/responses/{",
|
||||
"/responses/input_tokens",
|
||||
"/realtime/client_secrets",
|
||||
"/realtime/calls",
|
||||
"/realtime/transcription_sessions",
|
||||
"/realtime/translations/client_secrets",
|
||||
"/realtime/translations/calls",
|
||||
)
|
||||
) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models")
|
||||
concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model")
|
||||
assert managed_agent_route_allowed(concrete, None) is not unsupported, route
|
||||
|
|
|
|||
|
|
@ -818,6 +818,52 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route):
|
|||
assert RouteChecks.is_management_route(route=route) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route",
|
||||
[
|
||||
"/realtime/translations",
|
||||
"/v1/realtime/translations",
|
||||
"/realtime/translations/client_secrets",
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
"/realtime/translations/calls",
|
||||
"/v1/realtime/translations/calls",
|
||||
],
|
||||
)
|
||||
def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route):
|
||||
valid_token = UserAPIKeyAuth(
|
||||
user_id="test_user",
|
||||
allowed_routes=["openai_routes"],
|
||||
)
|
||||
|
||||
assert RouteChecks.is_llm_api_route(route=route) is True
|
||||
assert RouteChecks.is_management_route(route=route) is False
|
||||
assert (
|
||||
RouteChecks.is_virtual_key_allowed_to_call_route(
|
||||
route=route,
|
||||
valid_token=valid_token,
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route",
|
||||
[
|
||||
"/openai/v1/realtime/translations",
|
||||
"/openai/v1/realtime/translations/client_secrets",
|
||||
"/openai/v1/realtime/translations/calls",
|
||||
],
|
||||
)
|
||||
def test_openai_prefixed_realtime_translation_routes_denied_for_openai_virtual_keys(route: str) -> None:
|
||||
valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes"])
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token)
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
assert f"Tried to call route: {route}" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied():
|
||||
"""Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group"""
|
||||
|
||||
|
|
|
|||
|
|
@ -931,6 +931,43 @@ async def test_user_api_key_auth_live_websocket_ignores_query_model():
|
|||
mock_user_api_key_auth.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("auth_name", "path"),
|
||||
[("user_api_key_auth_websocket", "/v1/responses"), ("user_api_key_auth_realtime_websocket", "/v1/realtime")],
|
||||
)
|
||||
async def test_websocket_without_a_model_is_refused_for_a_role_limited_to_other_models(
|
||||
auth_name: str, path: str
|
||||
) -> None:
|
||||
from litellm.proxy._types import RoleBasedPermissions, UserAPIKeyAuth
|
||||
from litellm.proxy.auth import user_api_key_auth as auth_module
|
||||
from litellm.proxy.auth.handle_jwt import JWTAuthManager
|
||||
|
||||
async def role_limited_auth(request, api_key: str) -> UserAPIKeyAuth:
|
||||
JWTAuthManager.can_rbac_role_call_model(
|
||||
rbac_role=LitellmUserRoles.INTERNAL_USER,
|
||||
general_settings={
|
||||
"role_permissions": [RoleBasedPermissions(role=LitellmUserRoles.INTERNAL_USER, models=["chat"])]
|
||||
},
|
||||
model=json.loads(await request.body())["model"],
|
||||
)
|
||||
return UserAPIKeyAuth(api_key=api_key)
|
||||
|
||||
websocket: Final = MagicMock(spec=WebSocket)
|
||||
websocket.query_params = {}
|
||||
websocket.headers = {"authorization": "Bearer some_api_key"}
|
||||
websocket.scope = {"type": "websocket", "path": path, "headers": [(b"authorization", b"Bearer some_api_key")]}
|
||||
websocket.url = URL(url=path)
|
||||
websocket.close = AsyncMock()
|
||||
|
||||
with patch("litellm.proxy.auth.user_api_key_auth.user_api_key_auth", side_effect=role_limited_auth):
|
||||
with pytest.raises(HTTPException) as raised:
|
||||
await getattr(auth_module, auth_name)(websocket)
|
||||
|
||||
assert raised.value.status_code == 403
|
||||
websocket.close.assert_awaited_once_with(code=status.WS_1008_POLICY_VIOLATION)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accepted_websocket_reauthorizes_the_frame_model_without_closing():
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
|
|
|||
|
|
@ -12872,6 +12872,188 @@ async def test_websocket_auth_hands_the_reservation_to_the_socket_state():
|
|||
assert websocket.scope["state"]["budget_reservation"] is reservation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/realtime/translations", "/v1/realtime/translations"))
|
||||
@pytest.mark.parametrize("query", (b"", b"model=gpt-realtime-translate"), ids=("default_model", "explicit_model"))
|
||||
async def test_translation_websocket_auth_enforces_the_default_model_budget(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str, query: bytes
|
||||
) -> None:
|
||||
from fastapi import WebSocket
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket
|
||||
|
||||
model: Final = "gpt-realtime-translate"
|
||||
token: Final = UserAPIKeyAuth(
|
||||
token="test-key",
|
||||
models=[model],
|
||||
model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}},
|
||||
)
|
||||
limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0})
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await limiter.is_key_within_model_budget(user_api_key_dict=token, model=model)
|
||||
|
||||
async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
return token
|
||||
|
||||
attrs: Final = {
|
||||
**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate),
|
||||
"prisma_client": None,
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
for name, value in attrs.items():
|
||||
monkeypatch.setattr(proxy_server, name, value)
|
||||
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False)
|
||||
websocket: Final = WebSocket(
|
||||
scope={
|
||||
"type": "websocket",
|
||||
"path": path,
|
||||
"headers": [(b"authorization", b"Bearer sk-test")],
|
||||
"query_string": query,
|
||||
},
|
||||
receive=AsyncMock(),
|
||||
send=AsyncMock(),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
await user_api_key_auth_realtime_websocket(websocket)
|
||||
|
||||
assert denied.value.status_code == 403
|
||||
assert model in str(denied.value.detail)
|
||||
assert "budget" in str(denied.value.detail).lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/realtime/translations/client_secrets", "/v1/realtime/translations/client_secrets"))
|
||||
async def test_translation_client_secret_auth_enforces_the_session_model_budget(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
model: Final = "gpt-realtime-translate"
|
||||
token: Final = UserAPIKeyAuth(
|
||||
token="test-key",
|
||||
models=[model],
|
||||
model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}},
|
||||
)
|
||||
limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0})
|
||||
|
||||
async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
return token
|
||||
|
||||
attrs: Final = {
|
||||
**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate),
|
||||
"prisma_client": None,
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
for name, value in attrs.items():
|
||||
monkeypatch.setattr(proxy_server, name, value)
|
||||
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False)
|
||||
body: Final = json.dumps({"session": {"type": "translation", "model": model}}).encode()
|
||||
request: Final = Request(
|
||||
scope={
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
"headers": [(b"authorization", b"Bearer sk-test"), (b"content-type", b"application/json")],
|
||||
"query_string": b"",
|
||||
},
|
||||
receive=AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}),
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as denied:
|
||||
await user_api_key_auth(request=request, api_key="Bearer sk-test")
|
||||
|
||||
assert denied.value.type == "budget_exceeded"
|
||||
assert model in denied.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/realtime", "/v1/realtime", "/openai/v1/realtime"))
|
||||
async def test_managed_agent_transcription_websocket_without_a_model_is_refused(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
from fastapi import WebSocket
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket
|
||||
from litellm.types.agents import AgentResponse
|
||||
|
||||
token: Final = UserAPIKeyAuth(token="test-key")
|
||||
token.managed_agent_policy = AgentResponse.model_validate(
|
||||
{"agent_id": "agent", "agent_name": "Agent", "agent_card_params": {}, "execution_mode": "both"}
|
||||
)
|
||||
|
||||
async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
return token
|
||||
|
||||
attrs: Final = {**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), "prisma_client": None}
|
||||
for name, value in attrs.items():
|
||||
monkeypatch.setattr(proxy_server, name, value)
|
||||
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False)
|
||||
websocket: Final = WebSocket(
|
||||
scope={
|
||||
"type": "websocket",
|
||||
"path": path,
|
||||
"headers": [(b"authorization", b"Bearer sk-test")],
|
||||
"query_string": b"intent=transcription",
|
||||
},
|
||||
receive=AsyncMock(),
|
||||
send=AsyncMock(),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
await user_api_key_auth_realtime_websocket(websocket)
|
||||
|
||||
assert denied.value.status_code == 403
|
||||
assert "Managed inference requires an explicit or configured model" in str(denied.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ("/openai_passthrough/v1/realtime", "/v1/responses", "/vertex_ai/live"))
|
||||
async def test_non_realtime_websocket_auth_checks_no_default_transcription_model(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
from fastapi import WebSocket
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket
|
||||
|
||||
exhausted_model: Final = "gpt-realtime-whisper"
|
||||
token: Final = UserAPIKeyAuth(
|
||||
token="test-key",
|
||||
model_max_budget={exhausted_model: {"max_budget": 5.0, "budget_duration": "1d"}},
|
||||
)
|
||||
limiter, _ = await _real_model_budget_limiter(key_spend={exhausted_model: 10.0})
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await limiter.is_key_within_model_budget(user_api_key_dict=token, model=exhausted_model)
|
||||
|
||||
async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
return token
|
||||
|
||||
attrs: Final = {
|
||||
**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate),
|
||||
"prisma_client": None,
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
for name, value in attrs.items():
|
||||
monkeypatch.setattr(proxy_server, name, value)
|
||||
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False)
|
||||
websocket: Final = WebSocket(
|
||||
scope={
|
||||
"type": "websocket",
|
||||
"path": path,
|
||||
"headers": [(b"authorization", b"Bearer sk-test")],
|
||||
"query_string": b"intent=transcription",
|
||||
},
|
||||
receive=AsyncMock(),
|
||||
send=AsyncMock(),
|
||||
)
|
||||
|
||||
assert (await user_api_key_auth_websocket(websocket)).token == token.token
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admission_and_budget_reservation_read_the_key_spend_counter_with_one_redis_mget():
|
||||
from fastapi import Request
|
||||
|
|
|
|||
|
|
@ -786,6 +786,17 @@ async def test_get_form_data():
|
|||
assert result["temperature"] == "0.2"
|
||||
assert result["include"] == ["logprobs"]
|
||||
assert result["timestamp_granularities"] == ["word", "segment"]
|
||||
assert tuple(result) == (
|
||||
"file",
|
||||
"model",
|
||||
"include",
|
||||
"language",
|
||||
"prompt",
|
||||
"response_format",
|
||||
"stream",
|
||||
"temperature",
|
||||
"timestamp_granularities",
|
||||
)
|
||||
|
||||
|
||||
def test_get_tags_from_request_body_with_metadata_tags():
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from typing import Any, cast
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Final, cast
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy.common_utils.realtime_utils import realtime_request_body
|
||||
from litellm.proxy.proxy_server import _realtime_query_params_template
|
||||
|
|
@ -40,6 +42,13 @@ def test_realtime_request_body_caches_each_model_separately():
|
|||
assert gpt4o_body_first is not gpt4o_mini_body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", (None, "requested", 'model"with\\quotes', "model\nwith-newline"))
|
||||
def test_realtime_request_body_preserves_model_names(model: str | None) -> None:
|
||||
body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(model))
|
||||
|
||||
assert body["model"] == model
|
||||
|
||||
|
||||
def test_realtime_query_params_template_caches_each_pair_separately():
|
||||
params_with_intent_first = _realtime_query_params_template("gpt-4o", "intent-a")
|
||||
params_with_intent_second = _realtime_query_params_template("gpt-4o", "intent-a")
|
||||
|
|
|
|||
|
|
@ -10,17 +10,35 @@ Pins (PR2):
|
|||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import wave
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from openai import AsyncOpenAI
|
||||
from starlette.datastructures import FormData, UploadFile
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
from litellm.proxy.route_llm_request import route_request
|
||||
from litellm.proxy.utils import _failure_fields_to_lift
|
||||
from litellm.types.guardrails import GuardrailEventHooks, GuardrailStreamScope
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
|
||||
_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']"
|
||||
|
||||
|
|
@ -125,9 +143,7 @@ def patched_transcription(monkeypatch):
|
|||
return data
|
||||
|
||||
monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data)
|
||||
monkeypatch.setattr(
|
||||
proxy_server, "check_file_size_under_limit", lambda **kwargs: True
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True)
|
||||
|
||||
async def _form_data(request):
|
||||
from starlette.datastructures import FormData, UploadFile
|
||||
|
|
@ -159,6 +175,47 @@ def patched_transcription_error(monkeypatch, patched_transcription):
|
|||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched_transcription_stream(monkeypatch, patched_transcription):
|
||||
class _FakeEvent:
|
||||
def model_dump_json(self, exclude_unset: bool = False) -> str:
|
||||
return '{"type":"transcript.text.done","text":"hello world"}'
|
||||
|
||||
class _FakeAsyncStream:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
def __aiter__(self):
|
||||
async def _events():
|
||||
yield _FakeEvent()
|
||||
|
||||
return _events()
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
async def _form_data(request):
|
||||
from starlette.datastructures import FormData, UploadFile
|
||||
|
||||
upload = UploadFile(
|
||||
filename="audio.mp3",
|
||||
file=io.BytesIO(b"\x00\x01\x02"),
|
||||
)
|
||||
return FormData([("file", upload), ("model", "gpt-transcribe"), ("stream", "true")])
|
||||
|
||||
stream = _FakeAsyncStream()
|
||||
|
||||
async def _llm_call():
|
||||
return stream
|
||||
|
||||
async def _fake_route_request(*args, **kwargs):
|
||||
return _llm_call()
|
||||
|
||||
monkeypatch.setattr(proxy_server, "get_form_data", _form_data)
|
||||
monkeypatch.setattr(proxy_server, "route_request", _fake_route_request)
|
||||
yield stream
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"])
|
||||
def test_audio_speech_happy_path(client, auth_as, patched_speech, path):
|
||||
"""Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (happy)."""
|
||||
|
|
@ -359,3 +416,571 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error,
|
|||
response = client.post(path, files=files, data=data)
|
||||
assert response.status_code == 500
|
||||
assert len(response.content) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
|
||||
def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcription_stream, path):
|
||||
files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")}
|
||||
data = {"model": "gpt-transcribe", "stream": "true"}
|
||||
with auth_as():
|
||||
response = client.post(path, files=files, data=data)
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\ndata: [DONE]\n\n'
|
||||
assert patched_transcription_stream.closed is True
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
@pytest.mark.parametrize("configuration", ["default", "policy"])
|
||||
def test_streaming_transcription_keeps_streaming_with_output_guardrails(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configuration: str,
|
||||
) -> None:
|
||||
guardrail: Final = CustomGuardrail(
|
||||
guardrail_name="transcription-output",
|
||||
event_hook=GuardrailEventHooks.post_call,
|
||||
default_on=configuration == "default",
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
if configuration == "policy":
|
||||
pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")])
|
||||
proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: {
|
||||
**kwargs["data"],
|
||||
"metadata": {"_guardrail_pipelines": [("transcription-policy", pipeline)]},
|
||||
}
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
assert '"text":"hello world"' in response.text
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
def test_streamed_transcription_returns_the_litellm_call_and_model_headers(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
content=b'data: {"type":"transcript.text.done","text":"hello world"}\n\n',
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
async def route_to_the_sdk(
|
||||
data: Mapping[str, object], route_type: str, llm_router: object, user_model: object
|
||||
) -> object:
|
||||
return litellm.atranscription(
|
||||
model="openai/gpt-transcribe",
|
||||
file=data["file"],
|
||||
stream=True,
|
||||
client=upstream,
|
||||
litellm_call_id=data["litellm_call_id"],
|
||||
model_info={"id": "transcription-deployment"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
headers={"x-litellm-call-id": "streamed-transcription-call"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert '"text":"hello world"' in response.text
|
||||
assert response.headers["x-litellm-call-id"] == "streamed-transcription-call"
|
||||
assert response.headers["x-litellm-model-id"] == "transcription-deployment"
|
||||
assert [name for name in response.headers if name.startswith("x-litellm-response-cost")] == []
|
||||
proxy_server.proxy_logging_obj.update_request_status.assert_called_once_with(
|
||||
litellm_call_id="streamed-transcription-call", status="success"
|
||||
)
|
||||
|
||||
|
||||
class _TranscriptRedactor(CustomGuardrail):
|
||||
def __init__(self, stream_scope: GuardrailStreamScope = "both") -> None:
|
||||
super().__init__(
|
||||
guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call, stream_scope=stream_scope
|
||||
)
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self, data: dict[str, object], user_api_key_dict: object, response: object
|
||||
) -> None:
|
||||
if isinstance(response, TranscriptionResponse):
|
||||
response.text = "[REDACTED]"
|
||||
|
||||
|
||||
def _streams_when_asked(request: httpx.Request) -> httpx.Response:
|
||||
if b'name="stream"' in request.content:
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=b'data: {"type":"transcript.text.done","text":"my pin is 1234"}\n\n',
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
return httpx.Response(200, json={"text": "my pin is 1234", "usage": {"type": "duration", "seconds": 2}})
|
||||
|
||||
|
||||
def _sse_events(body: str) -> tuple[object, ...]:
|
||||
assert body.endswith("data: [DONE]\n\n")
|
||||
return tuple(json.loads(frame.removeprefix("data: ")) for frame in body.removesuffix("data: [DONE]\n\n").split("\n\n") if frame)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
@pytest.mark.parametrize("deployment_guardrails", (["transcript-redactor"], None))
|
||||
def test_streamed_transcription_honors_a_deployment_post_call_guardrail(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
deployment_guardrails: list[str] | None,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()])
|
||||
respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(side_effect=_streams_when_asked)
|
||||
deployment_params: Final = {
|
||||
"model": "openai/gpt-transcribe",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://transcribe-upstream.example/v1",
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-transcribe",
|
||||
"litellm_params": (
|
||||
{**deployment_params, "guardrails": deployment_guardrails}
|
||||
if deployment_guardrails
|
||||
else deployment_params
|
||||
),
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_request)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
if deployment_guardrails:
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
assert _sse_events(response.text) == (
|
||||
{"type": "transcript.text.delta", "delta": "[REDACTED]"},
|
||||
{"type": "transcript.text.done", "text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}},
|
||||
)
|
||||
else:
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
assert '"text":"my pin is 1234"' in response.text
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
@pytest.mark.parametrize(
|
||||
"stream_scope,upstream_streamed,expected_text",
|
||||
[("streaming", False, "[REDACTED]"), ("non_streaming", True, "my pin is 1234")],
|
||||
)
|
||||
def test_streamed_transcription_applies_a_deployment_guardrail_only_in_its_stream_scope(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
stream_scope: GuardrailStreamScope,
|
||||
upstream_streamed: bool,
|
||||
expected_text: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor(stream_scope=stream_scope)])
|
||||
upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(
|
||||
side_effect=_streams_when_asked
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-transcribe",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-transcribe",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://transcribe-upstream.example/v1",
|
||||
"guardrails": ["transcript-redactor"],
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_request)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
assert (b'name="stream"' in upstream_route.calls.last.request.content) is upstream_streamed
|
||||
assert _sse_events(response.text)[-1]["text"] == expected_text
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
def test_cached_guarded_transcription_still_streams_to_a_stream_client(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()])
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type="local"))
|
||||
upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(
|
||||
side_effect=_streams_when_asked
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-transcribe",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-transcribe",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://transcribe-upstream.example/v1",
|
||||
"guardrails": ["transcript-redactor"],
|
||||
},
|
||||
}
|
||||
],
|
||||
cache_responses=True,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_request)
|
||||
|
||||
with auth_as():
|
||||
responses: Final = [
|
||||
client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
|
||||
assert upstream_route.call_count == 1
|
||||
assert [response.headers["content-type"].split(";")[0] for response in responses] == ["text/event-stream"] * 2
|
||||
assert _sse_events(responses[1].text)[-1] == {
|
||||
"type": "transcript.text.done",
|
||||
"text": "[REDACTED]",
|
||||
"usage": {"type": "duration", "seconds": 2},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
def test_streamed_transcription_frames_keep_the_provider_event_shape(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
provider_events: Final = (
|
||||
{"type": "transcript.text.delta", "delta": "hello"},
|
||||
{"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 2}},
|
||||
)
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
content="".join(f"data: {json.dumps(event)}\n\n" for event in provider_events).encode(),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
async def route_to_the_sdk(
|
||||
data: Mapping[str, object], route_type: str, llm_router: object, user_model: object
|
||||
) -> object:
|
||||
return litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=data["file"], stream=data["stream"], client=upstream
|
||||
)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert _sse_events(response.text) == provider_events
|
||||
|
||||
|
||||
class _FirstDeltaThenReset(httpx.AsyncByteStream):
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
|
||||
raise httpx.ReadError("connection reset")
|
||||
|
||||
|
||||
def _silent_wav(seconds: int) -> bytes:
|
||||
buffer: Final = io.BytesIO()
|
||||
with wave.open(buffer, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(16_000)
|
||||
wav.writeframes(b"\x00\x00" * 16_000 * seconds)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription")
|
||||
def test_a_streamed_transcription_that_fails_mid_transcript_bills_the_uploaded_seconds(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenReset()
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
async def route_to_the_sdk(
|
||||
data: Mapping[str, object], route_type: str, llm_router: object, user_model: object
|
||||
) -> object:
|
||||
return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream)
|
||||
|
||||
four_second_upload: Final = _silent_wav(4)
|
||||
|
||||
async def stream_form(request: object) -> FormData:
|
||||
return FormData(
|
||||
[
|
||||
("file", UploadFile(filename="sample.wav", file=io.BytesIO(four_second_upload))),
|
||||
("model", "gpt-transcribe"),
|
||||
("stream", "true"),
|
||||
]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "get_form_data", stream_form)
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", four_second_upload, "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert '"error"' in response.text
|
||||
failure_request: Final = proxy_server.proxy_logging_obj.post_call_failure_hook.await_args.kwargs["request_data"]
|
||||
billed_cost: Final = _failure_fields_to_lift(failure_request)["response_cost"]
|
||||
assert isinstance(billed_cost, float)
|
||||
assert math.isclose(billed_cost, 4 * litellm.model_cost["gpt-transcribe"]["input_cost_per_second"])
|
||||
|
||||
|
||||
async def _scribe_stream_form(request: object) -> FormData:
|
||||
return FormData(
|
||||
[
|
||||
("file", UploadFile(filename="sample.wav", file=io.BytesIO(b"audio"))),
|
||||
("model", "scribe"),
|
||||
("stream", "true"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription")
|
||||
def test_transcription_stream_form_field_reaches_other_providers_as_sent(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
upstream: Final = respx_mock.post("https://elevenlabs.example/v1/speech-to-text").mock(
|
||||
return_value=httpx.Response(200, json={"text": "hello world", "language_code": "en"})
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "scribe",
|
||||
"litellm_params": {
|
||||
"model": "elevenlabs/scribe_v1",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://elevenlabs.example",
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_request)
|
||||
monkeypatch.setattr(proxy_server, "get_form_data", _scribe_stream_form)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "scribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["text"] == "hello world"
|
||||
assert b'name="stream"\r\n\r\ntrue\r\n' in upstream.calls.last.request.content
|
||||
|
||||
|
||||
def _streams_chat_and_transcription(request: httpx.Request) -> httpx.Response:
|
||||
sse_body: Final = (
|
||||
b'data: {"id":"chat-1","object":"chat.completion.chunk","created":0,"model":"gpt-4o",'
|
||||
b'"choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n'
|
||||
if request.url.path.endswith("/chat/completions")
|
||||
else b'data: {"type":"transcript.text.done","text":"hello world"}\n\n'
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=sse_body,
|
||||
headers={"content-type": "text/event-stream", "x-ratelimit-remaining-requests": "99"},
|
||||
)
|
||||
|
||||
|
||||
def _litellm_header_names(response: httpx.Response) -> frozenset[str]:
|
||||
return frozenset(
|
||||
name
|
||||
for name in response.headers
|
||||
if name.startswith("x-litellm-") and not name.startswith("x-litellm-response-cost")
|
||||
)
|
||||
|
||||
|
||||
def test_streamed_transcription_carries_the_headers_a_streamed_chat_completion_carries(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.route(host="stream-upstream.example").mock(side_effect=_streams_chat_and_transcription)
|
||||
upstream: Final = {"api_key": "sk-test", "api_base": "https://stream-upstream.example/v1"}
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", **upstream}},
|
||||
{"model_name": "gpt-transcribe", "litellm_params": {"model": "openai/gpt-transcribe", **upstream}},
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
with auth_as():
|
||||
chat: Final = client.post(
|
||||
"/v1/chat/completions",
|
||||
json={"model": "gpt-4o", "stream": True, "messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
transcription: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
assert chat.status_code == 200
|
||||
assert transcription.status_code == 200
|
||||
assert transcription.headers["content-type"].startswith("text/event-stream")
|
||||
assert _litellm_header_names(chat) - _litellm_header_names(transcription) <= {"x-litellm-overhead-duration-ms"}
|
||||
assert transcription.headers["x-litellm-model-name"] == "openai/gpt-transcribe"
|
||||
assert transcription.headers["llm_provider-x-ratelimit-remaining-requests"] == "99"
|
||||
assert [name for name in transcription.headers if name.startswith("x-litellm-response-cost")] == []
|
||||
|
||||
|
||||
class _FailsAfterTwoDeltas(httpx.AsyncByteStream):
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"hello "}\n\n'
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"world"}\n\n'
|
||||
raise httpx.ReadError("upstream connection dropped")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
def test_streamed_transcription_error_mid_stream_ends_with_an_error_event_and_runs_the_failure_hook(
|
||||
client: TestClient,
|
||||
auth_as: Callable[[], AbstractContextManager[None]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200, stream=_FailsAfterTwoDeltas(), headers={"content-type": "text/event-stream"}
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
async def route_to_the_sdk(
|
||||
data: Mapping[str, object], route_type: str, llm_router: object, user_model: object
|
||||
) -> object:
|
||||
return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream)
|
||||
|
||||
failure_hook: Final = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(proxy_server.proxy_logging_obj, "post_call_failure_hook", failure_hook)
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk)
|
||||
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/v1/audio/transcriptions",
|
||||
files={"file": ("sample.wav", b"audio", "audio/wav")},
|
||||
data={"model": "gpt-transcribe", "stream": "true"},
|
||||
)
|
||||
|
||||
data_lines: Final = tuple(
|
||||
line.removeprefix("data: ") for line in response.text.splitlines() if line.startswith("data: ")
|
||||
)
|
||||
frames: Final = tuple(json.loads(line) for line in data_lines[:-1])
|
||||
assert response.status_code == 200
|
||||
assert tuple(frame.get("delta") for frame in frames[:2]) == ("hello ", "world")
|
||||
assert len(frames) == 3
|
||||
assert data_lines[-1] == "[DONE]"
|
||||
assert "upstream connection dropped" in frames[2]["error"]["message"]
|
||||
assert set(frames[2]["error"]) == {"message", "type", "param", "code"}
|
||||
assert (frames[2]["error"]["type"], frames[2]["error"]["param"], frames[2]["error"]["code"]) == (
|
||||
"internal_server_error",
|
||||
None,
|
||||
"500",
|
||||
)
|
||||
failure_hook.assert_awaited_once()
|
||||
assert isinstance(failure_hook.await_args.kwargs["original_exception"], litellm.APIConnectionError)
|
||||
|
|
|
|||
|
|
@ -147,29 +147,20 @@ def test_get_litellm_model_cost_map_catalog_only_excludes_runtime_registered_ent
|
|||
)
|
||||
|
||||
|
||||
def test_get_litellm_model_cost_map_returns_cost_map():
|
||||
app = FastAPI()
|
||||
def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model_cost: Final = {
|
||||
"audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"},
|
||||
"token-model": {"input_cost_per_token": 0.01, "mode": "chat"},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", model_cost)
|
||||
app: Final = FastAPI()
|
||||
app.include_router(router)
|
||||
client = TestClient(app)
|
||||
client: Final = TestClient(app)
|
||||
|
||||
response = client.get("/public/litellm_model_cost_map")
|
||||
response: Final = client.get("/public/litellm_model_cost_map")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert isinstance(payload, dict)
|
||||
assert len(payload) > 0, "Expected model cost map to contain at least one model"
|
||||
|
||||
# Verify the structure contains expected keys for at least one model
|
||||
# Check for a common model like gpt-4 or gpt-3.5-turbo
|
||||
model_keys = list(payload.keys())
|
||||
assert len(model_keys) > 0
|
||||
|
||||
# Verify at least one model has expected cost fields
|
||||
sample_model = model_keys[0]
|
||||
sample_model_data = payload[sample_model]
|
||||
assert isinstance(sample_model_data, dict)
|
||||
# Check for common cost fields that should be present
|
||||
assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data
|
||||
assert response.json() == model_cost
|
||||
|
||||
|
||||
def test_public_ai_hub_info_is_public_by_default(monkeypatch):
|
||||
|
|
|
|||
|
|
@ -4,18 +4,23 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints:
|
|||
- POST /v1/realtime/calls
|
||||
"""
|
||||
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Awaitable
|
||||
from typing import Protocol
|
||||
from collections.abc import Awaitable, Mapping
|
||||
from typing import Final, Protocol
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.routing import Match
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE
|
||||
from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
||||
|
|
@ -23,8 +28,18 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
|||
encrypt_value_helper,
|
||||
)
|
||||
from litellm.proxy.realtime_endpoints.endpoints import (
|
||||
_ALLOWED_SESSION_TYPES,
|
||||
_coerce_realtime_session_type,
|
||||
_decode_realtime_token_payload,
|
||||
_encode_realtime_token_payload,
|
||||
_prepare_client_secret_session,
|
||||
)
|
||||
from litellm.types.realtime import (
|
||||
RealtimeAudioInputConfig,
|
||||
RealtimeAudioTranscriptionConfig,
|
||||
RealtimeClientSecretRequest,
|
||||
RealtimeSessionAudioConfig,
|
||||
RealtimeSessionConfig,
|
||||
)
|
||||
|
||||
# --- Unit tests: token encode/decode helpers ---
|
||||
|
|
@ -126,9 +141,7 @@ def mock_route_request_client_secrets():
|
|||
future_expires_at = int(time.time()) + 3600
|
||||
mock_resp = MagicMock(spec=httpx.Response)
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = (
|
||||
f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'
|
||||
)
|
||||
mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'
|
||||
mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode()
|
||||
mock_resp.headers = {}
|
||||
mock_resp.json.return_value = {
|
||||
|
|
@ -215,9 +228,7 @@ async def test_client_secrets_success_with_mock(
|
|||
mock_pre_call_hook,
|
||||
):
|
||||
"""POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream."""
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user", team_id="test-team"
|
||||
)
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team")
|
||||
try:
|
||||
client = TestClient(proxy_app)
|
||||
with (
|
||||
|
|
@ -275,13 +286,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model(
|
|||
"session": {
|
||||
"type": "transcription",
|
||||
"model": "gpt-4o-realtime-preview",
|
||||
"audio": {
|
||||
"input": {
|
||||
"transcription": {
|
||||
"model": "gpt-realtime-whisper"
|
||||
}
|
||||
}
|
||||
},
|
||||
"audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -312,12 +317,8 @@ async def test_client_secrets_transcription_routes_on_nested_model(
|
|||
async def _inner():
|
||||
resp = MagicMock(spec=httpx.Response)
|
||||
resp.status_code = 200
|
||||
resp.text = (
|
||||
f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'
|
||||
)
|
||||
resp.content = (
|
||||
f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'
|
||||
).encode()
|
||||
resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'
|
||||
resp.content = (f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}').encode()
|
||||
resp.headers = {}
|
||||
resp.json.return_value = {
|
||||
"value": "upstream_ephemeral_key",
|
||||
|
|
@ -351,13 +352,7 @@ async def test_client_secrets_transcription_routes_on_nested_model(
|
|||
"session": {
|
||||
"type": "transcription",
|
||||
"model": "gpt-4o-realtime-preview",
|
||||
"audio": {
|
||||
"input": {
|
||||
"transcription": {
|
||||
"model": "gpt-realtime-whisper"
|
||||
}
|
||||
}
|
||||
},
|
||||
"audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -367,10 +362,7 @@ async def test_client_secrets_transcription_routes_on_nested_model(
|
|||
session = captured["data"]["session"]
|
||||
assert session["type"] == "transcription"
|
||||
assert "model" not in session
|
||||
assert (
|
||||
session["audio"]["input"]["transcription"]["model"]
|
||||
== "gpt-realtime-whisper"
|
||||
)
|
||||
assert session["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper"
|
||||
encrypted_value = response.json()["value"]
|
||||
decoded = _decode_realtime_token_payload(
|
||||
decrypt_value_helper(
|
||||
|
|
@ -530,10 +522,7 @@ async def test_realtime_calls_replays_transcription_session_type(
|
|||
)
|
||||
|
||||
assert captured["session"]["type"] == "transcription"
|
||||
assert (
|
||||
captured["session"]["audio"]["input"]["transcription"]["model"]
|
||||
== "gpt-realtime-whisper"
|
||||
)
|
||||
assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper"
|
||||
|
||||
|
||||
# --- transcription_sessions endpoint ---
|
||||
|
|
@ -587,8 +576,12 @@ def test_transcription_sessions_requires_auth(proxy_app):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("body", [
|
||||
{"input_audio_transcription": {"model": "gpt-realtime-whisper"}},
|
||||
{"audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}},
|
||||
])
|
||||
async def test_transcription_sessions_rejects_disallowed_resolved_model(
|
||||
proxy_app,
|
||||
proxy_app, body,
|
||||
):
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user",
|
||||
|
|
@ -605,9 +598,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model(
|
|||
response = client.post(
|
||||
"/v1/realtime/transcription_sessions",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={
|
||||
"input_audio_transcription": {"model": "gpt-realtime-whisper"}
|
||||
},
|
||||
json=body,
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
|
@ -651,9 +642,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope(
|
|||
response = client.post(
|
||||
"/v1/realtime/transcription_sessions",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={
|
||||
"input_audio_transcription": {"model": "gpt-realtime-whisper"}
|
||||
},
|
||||
json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
|
@ -696,9 +685,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope(
|
|||
response = client.post(
|
||||
"/v1/realtime/transcription_sessions",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={
|
||||
"input_audio_transcription": {"model": "gpt-realtime-whisper"}
|
||||
},
|
||||
json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
|
@ -751,9 +738,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope
|
|||
response = client.post(
|
||||
"/v1/realtime/transcription_sessions",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={
|
||||
"input_audio_transcription": {"model": "gpt-realtime-whisper"}
|
||||
},
|
||||
json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}},
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
|
@ -947,13 +932,12 @@ async def test_transcription_sessions_encrypts_client_secret(
|
|||
POST /v1/realtime/transcription_sessions returns 200 and the ephemeral key
|
||||
under client_secret.value must be encrypted (never the raw upstream key).
|
||||
"""
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user", team_id="test-team"
|
||||
)
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team")
|
||||
captured_route_type = {}
|
||||
|
||||
async def _capturing_route(*args, **kwargs):
|
||||
captured_route_type["route_type"] = kwargs.get("route_type")
|
||||
captured_route_type["session"] = kwargs["data"]["transcription_session"]
|
||||
return await mock_route_request_transcription_sessions(*args, **kwargs)
|
||||
|
||||
try:
|
||||
|
|
@ -976,6 +960,8 @@ async def test_transcription_sessions_encrypts_client_secret(
|
|||
"/v1/realtime/transcription_sessions",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={
|
||||
"model": "gpt-realtime-whisper",
|
||||
"audio": {"input": {"transcription": {"model": "gpt-live-transcribe", "language": "en"}}},
|
||||
"input_audio_format": "pcm16",
|
||||
"input_audio_transcription": {"model": "gpt-realtime-whisper"},
|
||||
},
|
||||
|
|
@ -993,16 +979,61 @@ async def test_transcription_sessions_encrypts_client_secret(
|
|||
assert decrypted is not None
|
||||
assert "upstream_ephemeral_key" in decrypted
|
||||
# Routed through the dedicated transcription_sessions route type.
|
||||
assert (
|
||||
captured_route_type["route_type"]
|
||||
== "acreate_realtime_transcription_session"
|
||||
)
|
||||
assert captured_route_type["route_type"] == "acreate_realtime_transcription_session"
|
||||
assert captured_route_type["session"]["input_audio_transcription"]["model"] == "gpt-realtime-whisper"
|
||||
assert captured_route_type["session"]["audio"]["input"]["transcription"] == {
|
||||
"model": "gpt-realtime-whisper", "language": "en",
|
||||
}
|
||||
finally:
|
||||
proxy_app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("body", [
|
||||
{"model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}},
|
||||
{"input_audio_transcription": {"model": "gpt-4o-realtime-preview"}, "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}},
|
||||
{"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "gpt-realtime-whisper"}},
|
||||
])
|
||||
async def test_transcription_sessions_route_the_allowed_model_over_a_nested_model_the_key_cannot_call(
|
||||
proxy_app,
|
||||
mock_route_request_transcription_sessions,
|
||||
mock_add_litellm_data,
|
||||
mock_pre_call_hook,
|
||||
body,
|
||||
):
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user", models=["gpt-4o-realtime-preview"]
|
||||
)
|
||||
route: Final = AsyncMock(side_effect=mock_route_request_transcription_sessions)
|
||||
|
||||
try:
|
||||
client = TestClient(proxy_app)
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.route_request", route),
|
||||
patch("litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data),
|
||||
patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_logging,
|
||||
):
|
||||
mock_logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook)
|
||||
mock_logging.post_call_failure_hook = AsyncMock()
|
||||
|
||||
response = client.post(
|
||||
"/v1/realtime/transcription_sessions",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json=body,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
route.assert_awaited_once()
|
||||
routed_session: Final = route.await_args.kwargs["data"]["transcription_session"]
|
||||
assert {
|
||||
routed_session.get("input_audio_transcription", {}).get("model"),
|
||||
routed_session.get("audio", {}).get("input", {}).get("transcription", {}).get("model"),
|
||||
} - {None} == {"gpt-4o-realtime-preview"}
|
||||
finally:
|
||||
proxy_app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
|
||||
def test_session_type_coerced_for_unknown_value():
|
||||
"""An unrecognized session_type in the token falls back to 'realtime'."""
|
||||
payload = _encode_realtime_token_payload(
|
||||
ephemeral_key="epk",
|
||||
model_id="gpt-4o",
|
||||
|
|
@ -1011,12 +1042,327 @@ def test_session_type_coerced_for_unknown_value():
|
|||
expires_at=None,
|
||||
session_type="INJECTED_TYPE",
|
||||
)
|
||||
# Force-deserialize and check the coercion that happens in proxy_realtime_calls.
|
||||
decoded = json.loads(payload)
|
||||
session_type = decoded.get("session_type") or "realtime"
|
||||
if session_type not in ("realtime", "transcription"):
|
||||
session_type = "realtime"
|
||||
assert session_type == "realtime"
|
||||
assert decoded["session_type"] == "INJECTED_TYPE"
|
||||
assert _coerce_realtime_session_type("INJECTED_TYPE") == "realtime"
|
||||
assert _coerce_realtime_session_type(None) == "realtime"
|
||||
for allowed_session_type in _ALLOWED_SESSION_TYPES:
|
||||
assert _coerce_realtime_session_type(allowed_session_type) == allowed_session_type
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_translation_client_secret_rejects_disallowed_nested_transcription_model() -> None:
|
||||
req = RealtimeClientSecretRequest(
|
||||
model="gpt-realtime-translate",
|
||||
session=RealtimeSessionConfig(
|
||||
type="translation",
|
||||
model="gpt-realtime-translate",
|
||||
audio=RealtimeSessionAudioConfig(
|
||||
input=RealtimeAudioInputConfig(
|
||||
transcription=RealtimeAudioTranscriptionConfig(model="gpt-live-transcribe"),
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available for this API key"):
|
||||
await _prepare_client_secret_session(
|
||||
req=req,
|
||||
user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]),
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
forced_session_type="translation",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_translation_client_secret_binds_authorized_nested_transcription_model() -> None:
|
||||
req = RealtimeClientSecretRequest(
|
||||
model="gpt-realtime-translate",
|
||||
session=RealtimeSessionConfig(
|
||||
type="translation",
|
||||
model="gpt-realtime-translate",
|
||||
audio=RealtimeSessionAudioConfig(
|
||||
input=RealtimeAudioInputConfig(
|
||||
transcription=RealtimeAudioTranscriptionConfig(model="gpt-realtime-whisper"),
|
||||
)
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
model, session_data, session_type = await _prepare_client_secret_session(
|
||||
req=req,
|
||||
user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]),
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
forced_session_type="translation",
|
||||
)
|
||||
|
||||
assert model == "gpt-realtime-translate"
|
||||
assert session_type == "translation"
|
||||
assert session_data is not None
|
||||
assert session_data["model"] == "gpt-realtime-translate"
|
||||
assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
(
|
||||
"/openai/v1/realtime/client_secrets",
|
||||
"/openai/v1/realtime/calls",
|
||||
"/openai/v1/realtime/translations/client_secrets",
|
||||
"/openai/v1/realtime/translations/calls",
|
||||
),
|
||||
)
|
||||
def test_openai_prefixed_realtime_http_paths_stay_with_the_openai_pass_through(
|
||||
monkeypatch: pytest.MonkeyPatch, path: str
|
||||
) -> None:
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import openai_proxy_route
|
||||
|
||||
monkeypatch.setenv("LITELLM_DISABLE_LAZY_ROUTES", "true")
|
||||
features = tuple(feature for feature in LAZY_FEATURES if feature.name in ("llm_passthrough", "realtime"))
|
||||
app = FastAPI()
|
||||
reserve_lazy_slot(app, "llm_passthrough", features=features)
|
||||
attach_lazy_features(app, features)
|
||||
scope = {"type": "http", "method": "POST", "path": path, "root_path": "", "headers": [], "query_string": b""}
|
||||
|
||||
with TestClient(app):
|
||||
resolved = next(route for route in app.router.routes if route.matches(scope)[0] is Match.FULL)
|
||||
|
||||
assert getattr(resolved, "endpoint", None) is openai_proxy_route
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
"/realtime/translations/client_secrets",
|
||||
],
|
||||
)
|
||||
def test_translation_client_secret_aliases_bind_token_family(
|
||||
proxy_app,
|
||||
mock_route_request_client_secrets,
|
||||
mock_add_litellm_data,
|
||||
mock_pre_call_hook,
|
||||
path,
|
||||
):
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user",
|
||||
models=["gpt-realtime-translate"],
|
||||
)
|
||||
captured = {}
|
||||
|
||||
async def capture_route(*args, **kwargs):
|
||||
captured["route_type"] = kwargs["route_type"]
|
||||
captured["data"] = kwargs["data"]
|
||||
return await mock_route_request_client_secrets(*args, **kwargs)
|
||||
|
||||
try:
|
||||
client = TestClient(proxy_app)
|
||||
with (
|
||||
patch( # test-quality-ok: endpoint test captures the proxy routing boundary
|
||||
"litellm.proxy.proxy_server.route_request", side_effect=capture_route
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates request metadata enrichment
|
||||
"litellm.proxy.proxy_server.add_litellm_data_to_request",
|
||||
side_effect=mock_add_litellm_data,
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj"
|
||||
) as logging,
|
||||
):
|
||||
logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook)
|
||||
logging.post_call_failure_hook = AsyncMock()
|
||||
response = client.post(
|
||||
path,
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={"model": "gpt-realtime-translate"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert captured["route_type"] == "acreate_realtime_translation_client_secret"
|
||||
assert captured["data"]["session"] == {
|
||||
"type": "translation",
|
||||
"model": "gpt-realtime-translate",
|
||||
}
|
||||
decrypted = decrypt_value_helper(
|
||||
response.json()["value"],
|
||||
key="client_secret.value",
|
||||
exception_type="debug",
|
||||
)
|
||||
decoded = _decode_realtime_token_payload(decrypted or "")
|
||||
assert decoded is not None
|
||||
assert decoded["session_type"] == "translation"
|
||||
finally:
|
||||
proxy_app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"/v1/realtime/translations/calls",
|
||||
"/realtime/translations/calls",
|
||||
],
|
||||
)
|
||||
def test_translation_calls_aliases_route_translation_session(
|
||||
proxy_app,
|
||||
mock_route_request_realtime_calls,
|
||||
mock_add_litellm_data,
|
||||
mock_pre_call_hook,
|
||||
path,
|
||||
):
|
||||
token = encrypt_value_helper(
|
||||
_encode_realtime_token_payload(
|
||||
ephemeral_key="ek_test",
|
||||
model_id="gpt-realtime-translate",
|
||||
user_id="test-user",
|
||||
team_id=None,
|
||||
expires_at=int(time.time()) + 3600,
|
||||
session_type="translation",
|
||||
)
|
||||
)
|
||||
captured = {}
|
||||
|
||||
async def capture_route(*args, **kwargs):
|
||||
captured["route_type"] = kwargs["route_type"]
|
||||
captured["data"] = kwargs["data"]
|
||||
return await mock_route_request_realtime_calls(*args, **kwargs)
|
||||
|
||||
client = TestClient(proxy_app)
|
||||
with (
|
||||
patch( # test-quality-ok: endpoint test captures the proxy routing boundary
|
||||
"litellm.proxy.proxy_server.route_request", side_effect=capture_route
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates request metadata enrichment
|
||||
"litellm.proxy.proxy_server.add_litellm_data_to_request",
|
||||
side_effect=mock_add_litellm_data,
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj"
|
||||
) as logging,
|
||||
):
|
||||
logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook)
|
||||
logging.post_call_failure_hook = AsyncMock()
|
||||
response = client.post(
|
||||
path,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
content=b"v=0\r\n",
|
||||
)
|
||||
|
||||
assert response.status_code == 201
|
||||
assert captured["route_type"] == "arealtime_translation_calls"
|
||||
assert captured["data"]["session"] == {
|
||||
"type": "translation",
|
||||
"model": "gpt-realtime-translate",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disable_global_guardrails", "expected_status", "expected_body"),
|
||||
[(True, 201, "v=0"), (False, 400, TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE)],
|
||||
)
|
||||
def test_translation_calls_honor_the_minting_keys_guardrail_opt_out(
|
||||
proxy_app: FastAPI,
|
||||
mock_route_request_client_secrets: object,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
disable_global_guardrails: bool,
|
||||
expected_status: int,
|
||||
expected_body: str,
|
||||
) -> None:
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-realtime-translate",
|
||||
"litellm_params": {"model": "openai/gpt-realtime-translate", "api_key": "sk-upstream"},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"callbacks",
|
||||
[CustomGuardrail(guardrail_name="global-pre-call", event_hook=GuardrailEventHooks.pre_call, default_on=True)],
|
||||
)
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user",
|
||||
models=["gpt-realtime-translate"],
|
||||
metadata={"disable_global_guardrails": disable_global_guardrails},
|
||||
)
|
||||
client: Final = TestClient(proxy_app, raise_server_exceptions=False)
|
||||
try:
|
||||
with (
|
||||
patch( # test-quality-ok: endpoint test stands in for the upstream client_secrets call
|
||||
"litellm.proxy.proxy_server.route_request", side_effect=mock_route_request_client_secrets
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates request metadata enrichment while minting
|
||||
"litellm.proxy.proxy_server.add_litellm_data_to_request",
|
||||
side_effect=mock_add_litellm_data,
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger while minting
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj"
|
||||
) as logging,
|
||||
):
|
||||
logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook)
|
||||
logging.post_call_failure_hook = AsyncMock()
|
||||
minted: Final = client.post(
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
headers={"Authorization": "Bearer sk-test-master-key"},
|
||||
json={"model": "gpt-realtime-translate"},
|
||||
)
|
||||
finally:
|
||||
proxy_app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.post("https://api.openai.com/v1/realtime/translations/calls").mock(
|
||||
return_value=httpx.Response(201, content=b"v=0\r\n", headers={"content-type": "application/sdp"})
|
||||
)
|
||||
answered: Final = client.post(
|
||||
"/v1/realtime/translations/calls",
|
||||
headers={"Authorization": f"Bearer {minted.json()['value']}"},
|
||||
content=b"v=0\r\n",
|
||||
)
|
||||
|
||||
assert minted.status_code == 200
|
||||
assert answered.status_code == expected_status
|
||||
assert expected_body in answered.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"session_type,path",
|
||||
[
|
||||
("realtime", "/v1/realtime/translations/calls"),
|
||||
("translation", "/v1/realtime/calls"),
|
||||
],
|
||||
)
|
||||
def test_realtime_calls_reject_cross_family_token(proxy_app, session_type, path):
|
||||
model = "gpt-realtime-translate" if session_type == "translation" else "gpt-realtime-2"
|
||||
token = encrypt_value_helper(
|
||||
_encode_realtime_token_payload(
|
||||
ephemeral_key="ek_test",
|
||||
model_id=model,
|
||||
user_id=None,
|
||||
team_id=None,
|
||||
expires_at=int(time.time()) + 3600,
|
||||
session_type=session_type,
|
||||
)
|
||||
)
|
||||
response = TestClient(proxy_app).post(
|
||||
path,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
content=b"v=0\r\n",
|
||||
)
|
||||
assert response.status_code == 401
|
||||
assert response.json()["error"] == "Token is not valid for this Realtime endpoint"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1142,9 +1488,7 @@ async def test_transcription_sessions_returns_upstream_error_verbatim(
|
|||
|
||||
return _inner()
|
||||
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user", team_id="test-team"
|
||||
)
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team")
|
||||
try:
|
||||
client = TestClient(proxy_app)
|
||||
with (
|
||||
|
|
@ -1184,9 +1528,7 @@ async def test_transcription_sessions_wraps_route_exception(
|
|||
async def _raise_http(*args, **kwargs):
|
||||
raise HTTPException(status_code=403, detail="Model not allowed")
|
||||
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="test-user"
|
||||
)
|
||||
proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user")
|
||||
try:
|
||||
client = TestClient(proxy_app, raise_server_exceptions=False)
|
||||
with (
|
||||
|
|
@ -1287,3 +1629,260 @@ def test_transcription_sessions_rejection_answers_an_openai_typed_error(
|
|||
|
||||
assert response.status_code == 400
|
||||
assert (response.json()["error"]["type"], response.json()["error"]["param"]) == ("invalid_request_error", None)
|
||||
|
||||
|
||||
_AZURE_REALTIME_DEPLOYMENTS: Final = (
|
||||
("azure-realtime-alias", "azure/qa-realtime-deployment"),
|
||||
("azure-transcribe-alias", "azure/qa-whisper-deployment"),
|
||||
("azure-translate-alias", "azure/qa-translate-deployment"),
|
||||
)
|
||||
|
||||
|
||||
def _post_through_the_azure_deployments(
|
||||
proxy_app: FastAPI,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
path: str,
|
||||
body: object,
|
||||
deployments: tuple[tuple[str, str], ...] = _AZURE_REALTIME_DEPLOYMENTS,
|
||||
) -> tuple[httpx.Response, respx.Route]:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
model_list: Final = [ # mutable-ok: Router takes a list of deployments
|
||||
{
|
||||
"model_name": alias,
|
||||
"litellm_params": {
|
||||
"model": deployment,
|
||||
"api_base": "https://realtime-qa.openai.azure.com",
|
||||
"api_key": "azure-test-key",
|
||||
},
|
||||
}
|
||||
for alias, deployment in deployments
|
||||
]
|
||||
monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list))
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", model_list)
|
||||
upstream: Final = respx_mock.route(host="realtime-qa.openai.azure.com").mock(
|
||||
return_value=httpx.Response(200, json={"value": "ek_azure", "expires_at": int(time.time()) + 3600})
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
proxy_app.dependency_overrides,
|
||||
user_api_key_auth,
|
||||
lambda: UserAPIKeyAuth(user_id="test-user", models=[alias for alias, _ in deployments]),
|
||||
)
|
||||
with (
|
||||
patch( # test-quality-ok: endpoint test isolates request metadata enrichment
|
||||
"litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data
|
||||
),
|
||||
patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger
|
||||
"litellm.proxy.proxy_server.proxy_logging_obj"
|
||||
) as logging,
|
||||
):
|
||||
logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook)
|
||||
logging.post_call_failure_hook = AsyncMock()
|
||||
response: Final = TestClient(proxy_app).post(
|
||||
path, headers={"Authorization": "Bearer sk-test-master-key"}, json=body
|
||||
)
|
||||
return response, upstream
|
||||
|
||||
|
||||
def _upstream_session_value(upstream: respx.Route, session_path: tuple[str, ...]) -> object:
|
||||
return functools.reduce(
|
||||
lambda node, key: node[key], session_path, json.loads(upstream.calls.last.request.content)["session"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "body", "upstream_model_path", "expected_model"),
|
||||
(
|
||||
(
|
||||
"/v1/realtime/client_secrets",
|
||||
{"session": {"type": "realtime", "model": "azure-realtime-alias"}},
|
||||
("model",),
|
||||
"qa-realtime-deployment",
|
||||
),
|
||||
(
|
||||
"/v1/realtime/client_secrets",
|
||||
{"model": "azure-realtime-alias"},
|
||||
("model",),
|
||||
"qa-realtime-deployment",
|
||||
),
|
||||
(
|
||||
"/v1/realtime/client_secrets",
|
||||
{
|
||||
"session": {
|
||||
"type": "transcription",
|
||||
"audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}},
|
||||
}
|
||||
},
|
||||
("audio", "input", "transcription", "model"),
|
||||
"qa-whisper-deployment",
|
||||
),
|
||||
(
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
{"model": "azure-translate-alias"},
|
||||
("model",),
|
||||
"qa-translate-deployment",
|
||||
),
|
||||
(
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
{
|
||||
"model": "azure-translate-alias",
|
||||
"session": {"audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}},
|
||||
},
|
||||
("audio", "input", "transcription", "model"),
|
||||
"qa-whisper-deployment",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_client_secrets_mint_a_secret_for_the_azure_deployment_model(
|
||||
proxy_app: FastAPI,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
path: str,
|
||||
body: dict[str, object],
|
||||
upstream_model_path: tuple[str, ...],
|
||||
expected_model: str,
|
||||
) -> None:
|
||||
response, upstream = _post_through_the_azure_deployments(
|
||||
proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _upstream_session_value(upstream, upstream_model_path) == expected_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "session_path", "expected"),
|
||||
(
|
||||
(
|
||||
{
|
||||
"session": {
|
||||
"type": "realtime",
|
||||
"model": "azure-realtime-alias",
|
||||
"audio": {"input": {"turn_detection": None}},
|
||||
}
|
||||
},
|
||||
("audio", "input", "turn_detection"),
|
||||
None,
|
||||
),
|
||||
(
|
||||
{
|
||||
"session": {
|
||||
"type": "transcription",
|
||||
"audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}, "turn_detection": None}},
|
||||
}
|
||||
},
|
||||
("audio", "input", "turn_detection"),
|
||||
None,
|
||||
),
|
||||
({"session": {"type": "conversation", "model": "azure-realtime-alias"}}, ("type",), "realtime"),
|
||||
(
|
||||
{
|
||||
"session": {
|
||||
"type": "transcription",
|
||||
"audio": {"input": {"transcription": {"model": "azure-transcribe-alias", "delay": "auto"}}},
|
||||
}
|
||||
},
|
||||
("audio", "input", "transcription", "delay"),
|
||||
"auto",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_client_secrets_forward_every_session_value_the_caller_set(
|
||||
proxy_app: FastAPI,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
body: Mapping[str, object],
|
||||
session_path: tuple[str, ...],
|
||||
expected: object,
|
||||
) -> None:
|
||||
response, upstream = _post_through_the_azure_deployments(
|
||||
proxy_app,
|
||||
mock_add_litellm_data,
|
||||
mock_pre_call_hook,
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
"/v1/realtime/client_secrets",
|
||||
body,
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert _upstream_session_value(upstream, session_path) == expected
|
||||
|
||||
|
||||
def test_client_secrets_without_a_model_let_the_upstream_pick_its_default_through_a_wildcard_deployment(
|
||||
proxy_app: FastAPI,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, upstream = _post_through_the_azure_deployments(
|
||||
proxy_app,
|
||||
mock_add_litellm_data,
|
||||
mock_pre_call_hook,
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
"/v1/realtime/client_secrets",
|
||||
{"session": {"type": "realtime"}},
|
||||
deployments=(("*", "openai/*"),),
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(upstream.calls.last.request.content)["session"] == {"type": "realtime"}
|
||||
|
||||
|
||||
def test_client_secrets_send_the_deployment_model_in_the_flat_transcription_field(
|
||||
proxy_app: FastAPI,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response, upstream = _post_through_the_azure_deployments(
|
||||
proxy_app,
|
||||
mock_add_litellm_data,
|
||||
mock_pre_call_hook,
|
||||
respx_mock,
|
||||
monkeypatch,
|
||||
"/v1/realtime/client_secrets",
|
||||
{"session": {"type": "transcription", "input_audio_transcription": {"model": "azure-transcribe-alias"}}},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(upstream.calls.last.request.content)["session"] == {
|
||||
"type": "transcription",
|
||||
"input_audio_transcription": {"model": "qa-whisper-deployment"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "body"),
|
||||
(
|
||||
("/v1/realtime/client_secrets", {"session": "not-a-session"}),
|
||||
("/v1/realtime/translations/client_secrets", {"session": {"audio": {"input": "not-an-input"}}}),
|
||||
("/v1/realtime/transcription_sessions", {"audio": {"input": {"transcription": "not-a-transcription"}}}),
|
||||
),
|
||||
)
|
||||
def test_realtime_session_routes_answer_400_for_a_malformed_body(
|
||||
proxy_app: FastAPI,
|
||||
mock_add_litellm_data: AddLitellmDataToRequest,
|
||||
mock_pre_call_hook: PreCallHook,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
path: str,
|
||||
body: Mapping[str, object],
|
||||
) -> None:
|
||||
response, upstream = _post_through_the_azure_deployments(
|
||||
proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body
|
||||
)
|
||||
|
||||
assert (response.status_code, response.json()["error"]["type"]) == (400, "invalid_request_error")
|
||||
assert upstream.called is False
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import sys
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import replace
|
||||
from types import ModuleType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -443,3 +444,45 @@ def test_root_authorization_server_metadata_is_the_gateways_not_the_byok_connect
|
|||
assert root.json()["token_endpoint"] == "https://gateway.example.com/token"
|
||||
assert connector.status_code == 200
|
||||
assert connector.json()["token_endpoint"] == "https://gateway.example.com/v1/mcp/oauth/token"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_order",
|
||||
(("/realtime/unrouted", "/mcp/unrouted"), ("/mcp/unrouted", "/realtime/unrouted")),
|
||||
ids=("realtime_first", "realtime_last"),
|
||||
)
|
||||
def test_realtime_and_mcp_features_keep_every_route_once_in_either_load_order(
|
||||
monkeypatch: pytest.MonkeyPatch, load_order: tuple[str, str]
|
||||
) -> None:
|
||||
monkeypatch.delenv(FLAG, raising=False)
|
||||
|
||||
async def mcp_mount() -> Mapping[str, str]:
|
||||
return {"route": "mcp_mount"}
|
||||
|
||||
mounted_app: Final = FastAPI()
|
||||
mounted_app.add_api_route("/{path:path}", mcp_mount, methods=["GET"])
|
||||
mcp_module: Final = ModuleType("tests.unit.proxy.lazy_fixture_mcp_app")
|
||||
mcp_module.app = mounted_app # pyright: ignore[reportAttributeAccessIssue] # fixture module built at test time
|
||||
monkeypatch.setitem(sys.modules, mcp_module.__name__, mcp_module)
|
||||
features: Final = tuple(
|
||||
replace(feature, module_path=mcp_module.__name__) if feature.name == "mcp_app" else feature
|
||||
for feature in LAZY_FEATURES
|
||||
if feature.name in ("mcp_app", "realtime")
|
||||
)
|
||||
|
||||
async def mcp_proxy() -> Mapping[str, str]:
|
||||
return {"route": "mcp_proxy"}
|
||||
|
||||
app: Final = FastAPI()
|
||||
attach_lazy_features(app, features)
|
||||
app.add_api_route("/mcp/proxy", mcp_proxy, methods=["GET"])
|
||||
|
||||
with TestClient(app) as client:
|
||||
for path in load_order:
|
||||
client.get(path)
|
||||
response: Final = client.get("/mcp/proxy")
|
||||
|
||||
assert loaded_lazy_modules(app) == {feature.module_path for feature in features}
|
||||
assert response.json() == {"route": "mcp_proxy"}
|
||||
route_ids: Final = tuple(id(route) for route in app.routes)
|
||||
assert len(route_ids) == len(frozenset(route_ids))
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from fastapi import FastAPI, HTTPException, WebSocket
|
|||
|
||||
# test /chat/completion request to the proxy
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.testclient import WebSocketTestSession
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
|
|
@ -3661,6 +3662,91 @@ def test_live_session_pre_call_abort_releases_reservations(
|
|||
route_call.assert_not_awaited()
|
||||
|
||||
|
||||
def _receive_until_closed(websocket: WebSocketTestSession) -> None:
|
||||
while True:
|
||||
websocket.receive_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "expected_upstream_url"),
|
||||
(
|
||||
pytest.param(
|
||||
{"model": "openai/gateway-transcribe", "api_base": "https://gateway.example/v1"},
|
||||
"wss://gateway.example/v1/realtime?model=gateway-transcribe&intent=transcription",
|
||||
id="gateway",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "xai/grok-voice", "api_base": "https://api.x.ai/v1"},
|
||||
"wss://api.x.ai/v1/realtime?model=grok-voice&intent=transcription",
|
||||
id="xai",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "openai/gpt-live-transcribe", "api_base": "https://api.openai.com/v1"},
|
||||
"wss://api.openai.com/v1/realtime?intent=transcription",
|
||||
id="openai",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_realtime_transcription_websocket_keeps_the_model_upstream_except_on_openai_hosts(
|
||||
client_no_auth: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
litellm_params: Mapping[str, str],
|
||||
expected_upstream_url: str,
|
||||
) -> None:
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
"llm_router",
|
||||
litellm.Router(
|
||||
model_list=[{"model_name": "team-transcriber", "litellm_params": {**litellm_params, "api_key": "fake"}}]
|
||||
),
|
||||
)
|
||||
connect: Final = MagicMock(side_effect=RuntimeError("upstream url captured"))
|
||||
monkeypatch.setattr("websockets.connect", connect)
|
||||
|
||||
with (
|
||||
client_no_auth.websocket_connect(
|
||||
"/v1/realtime?model=team-transcriber&intent=transcription",
|
||||
headers={"Authorization": "Bearer test-api-key"},
|
||||
) as websocket,
|
||||
pytest.raises(WebSocketDisconnect),
|
||||
):
|
||||
_receive_until_closed(websocket)
|
||||
|
||||
assert connect.call_args.args[0] == expected_upstream_url
|
||||
|
||||
|
||||
def test_openai_prefixed_translation_websocket_takes_the_openai_websocket_pass_through(
|
||||
client_no_auth: TestClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _OPENAI_WS_DISABLED_REFUSAL
|
||||
|
||||
route_call: Final = AsyncMock()
|
||||
monkeypatch.setattr(proxy_server, "route_request", route_call)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setitem(proxy_server.app.dependency_overrides, user_api_key_auth_websocket, lambda: UserAPIKeyAuth())
|
||||
|
||||
with client_no_auth.websocket_connect(
|
||||
"/openai/v1/realtime/translations?model=gpt-realtime-translate",
|
||||
headers={"Authorization": "Bearer test-api-key"},
|
||||
) as websocket:
|
||||
refusal: Final = websocket.receive_json()
|
||||
with pytest.raises(WebSocketDisconnect) as closed:
|
||||
websocket.receive_json()
|
||||
|
||||
assert refusal["error"]["message"] == _OPENAI_WS_DISABLED_REFUSAL.message
|
||||
assert closed.value.code == 1008
|
||||
route_call.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/openai/v1/live/sessions", "/v1/live/sessions", "/live/sessions"])
|
||||
def test_live_session_invalid_start_is_rejected_without_routing(
|
||||
path: str,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from fastapi import FastAPI, HTTPException, Request
|
|||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm
|
||||
import litellm.proxy.proxy_server as proxy_server_module
|
||||
|
|
@ -11092,7 +11093,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory():
|
|||
|
||||
|
||||
def test_realtime_websocket_route_aliases_registered():
|
||||
"""Realtime sessions reach the proxy via three path aliases stacked on
|
||||
"""Realtime sessions reach the proxy via five path aliases stacked on
|
||||
`realtime_websocket_endpoint`. Dropping any of them silently 405s
|
||||
WebSocket upgrades because the catch-all `/openai/{endpoint:path}`
|
||||
HTTP passthrough only declares HTTP methods. The aliases must also be
|
||||
|
|
@ -11108,7 +11109,13 @@ def test_realtime_websocket_route_aliases_registered():
|
|||
websocket_paths = {route.path for route in app.routes if isinstance(route, WebSocketRoute)}
|
||||
openai_routes = LiteLLMRoutes.openai_routes.value
|
||||
|
||||
for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"):
|
||||
for expected in (
|
||||
"/openai/v1/realtime",
|
||||
"/v1/realtime",
|
||||
"/realtime",
|
||||
"/v1/realtime/translations",
|
||||
"/realtime/translations",
|
||||
):
|
||||
assert expected in websocket_paths, (
|
||||
f"{expected!r} missing from registered WebSocket routes; the "
|
||||
f"realtime endpoint will 405 for clients hitting this path."
|
||||
|
|
@ -11127,7 +11134,7 @@ def _lit6973_fake_realtime_ws() -> MagicMock:
|
|||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
ws.scope = {"headers": [], "type": "websocket"}
|
||||
ws.url = "ws://testserver/v1/realtime"
|
||||
ws.url = URL("ws://testserver/v1/realtime")
|
||||
ws.accept = AsyncMock()
|
||||
ws.send_text = AsyncMock()
|
||||
ws.close = AsyncMock()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,10 @@
|
|||
"""Tests for scripts/check_openapi_docs.py, the scan behind scripts/openapi_docs_gate.py."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import check_openapi_docs as checker
|
||||
|
||||
_BODY: Final = {"content": {"application/json": {"schema": {"type": "object"}}}}
|
||||
|
|
@ -9,8 +12,23 @@ _RESPONSES: Final = {"200": {"content": {"application/json": {"schema": {"$ref":
|
|||
_DOCUMENTED: Final = {"description": "Does a thing", "requestBody": _BODY, "responses": _RESPONSES}
|
||||
|
||||
|
||||
def test_a_fully_documented_post_has_no_findings() -> None:
|
||||
assert checker.violations({"paths": {"/v1/thing": {"post": _DOCUMENTED}}}) == ()
|
||||
def _post_with_media_types(request_media_type: str, response_media_type: str) -> Mapping[str, object]:
|
||||
return {
|
||||
**_DOCUMENTED,
|
||||
"requestBody": {"content": {request_media_type: {"schema": {"type": "string"}}}},
|
||||
"responses": {"200": {"content": {response_media_type: {"schema": {"type": "string"}}}}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("media_type", ("application/json", "application/sdp"))
|
||||
def test_a_fully_documented_post_has_no_findings(media_type: str) -> None:
|
||||
operation: Final = _post_with_media_types(media_type, media_type)
|
||||
assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == ()
|
||||
|
||||
|
||||
def test_a_schema_under_another_media_type_still_counts_as_undocumented() -> None:
|
||||
operation: Final = _post_with_media_types("multipart/form-data", "audio/wav")
|
||||
assert {v.code for v in checker.violations({"paths": {"/v1/thing": {"post": operation}}})} == {"OA001", "OA003"}
|
||||
|
||||
|
||||
def test_each_missing_piece_is_its_own_rule() -> None:
|
||||
|
|
|
|||
|
|
@ -1451,6 +1451,15 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type():
|
|||
assert _transcription_usage_cost({}, {}) == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("malformed_seconds", [-1.0, float("nan"), float("inf"), 10**1000, True])
|
||||
def test_transcription_ignores_invalid_provider_duration(malformed_seconds: float | int | bool) -> None:
|
||||
from litellm.cost_calculator import _get_transcription_usage_duration
|
||||
|
||||
response: Final = SimpleNamespace(usage={"type": "duration", "seconds": malformed_seconds})
|
||||
|
||||
assert _get_transcription_usage_duration(response) is None
|
||||
|
||||
|
||||
def test_get_transcription_model_falls_back_to_session_model(monkeypatch):
|
||||
"""session.model is used when transcription-specific model fields are absent."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
|
@ -7442,6 +7451,39 @@ def test_transcription_cost_falls_back_to_provider_billed_seconds(
|
|||
assert cost == pytest.approx(0.01 * expected_seconds)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider", "measured_seconds", "expected_seconds"),
|
||||
[("openai", 1.0, 1.0), ("azure", 1.0, 1.0), ("azure", None, 7.0)],
|
||||
ids=["openai_measured_upload_wins", "azure_measured_upload_wins", "unmeasured_upload_bills_provider_seconds"],
|
||||
)
|
||||
def test_transcription_priced_by_provider_duration_still_bills_the_measured_upload(
|
||||
monkeypatch: pytest.MonkeyPatch, provider: str, measured_seconds: float | None, expected_seconds: float
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"gpt-fake-transcribe",
|
||||
{
|
||||
"input_cost_per_second": 0.01,
|
||||
"litellm_provider": provider,
|
||||
"mode": "audio_transcription",
|
||||
"provider_specific_entry": {"transcription_usage_duration": 1},
|
||||
},
|
||||
)
|
||||
measured: Final = {} if measured_seconds is None else {"audio_transcription_duration": measured_seconds}
|
||||
transcription: Final = convert_to_model_response_object(
|
||||
response_object={"text": "What is the weather in Paris?", "usage": {"type": "duration", "seconds": 7}},
|
||||
model_response_object=TranscriptionResponse(),
|
||||
hidden_params={"model": "gpt-fake-transcribe", "custom_llm_provider": provider, **measured},
|
||||
response_type="audio_transcription",
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=transcription, model=f"{provider}/gpt-fake-transcribe", call_type="atranscription"
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(0.01 * expected_seconds)
|
||||
|
||||
|
||||
def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
|
|
@ -7466,6 +7508,77 @@ def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypat
|
|||
assert cost == pytest.approx(0.000075 * 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_id,deployment_rates,expected_cost",
|
||||
[
|
||||
("per-second-only", {"input_cost_per_second": 0.0001}, 30 * 0.0001),
|
||||
("zero-token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 0.0}, 0.0),
|
||||
("token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 1e-6}, 500 * 1e-6),
|
||||
],
|
||||
)
|
||||
def test_transcription_deployment_bills_seconds_only_without_a_declared_token_rate(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
deployment_id: str,
|
||||
deployment_rates: Mapping[str, float],
|
||||
expected_cost: float,
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
deployment_id,
|
||||
{**deployment_rates, "litellm_provider": "openai", "mode": "audio_transcription"},
|
||||
)
|
||||
transcription: Final = convert_to_model_response_object(
|
||||
response_object={
|
||||
"text": "What is the weather in Paris?",
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": 500,
|
||||
"output_tokens": 100,
|
||||
"total_tokens": 600,
|
||||
"input_token_details": {"audio_tokens": 500, "text_tokens": 0},
|
||||
},
|
||||
},
|
||||
model_response_object=TranscriptionResponse(),
|
||||
hidden_params={
|
||||
"model": "gpt-4o-transcribe",
|
||||
"custom_llm_provider": "openai",
|
||||
"audio_transcription_duration": 30.0,
|
||||
},
|
||||
response_type="audio_transcription",
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=transcription,
|
||||
model="openai/gpt-4o-transcribe",
|
||||
call_type="atranscription",
|
||||
custom_pricing=True,
|
||||
router_model_id=deployment_id,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"])
|
||||
def test_azure_transcription_without_usage_bills_only_tokens_while_openai_bills_the_measured_audio(
|
||||
_local_model_cost_map: None, model: str
|
||||
) -> None:
|
||||
def cost_of_ten_seconds_without_usage(provider: str) -> float:
|
||||
transcript: Final = TranscriptionResponse(text="Good")
|
||||
transcript.set_audio_transcription_duration(10.0)
|
||||
return completion_cost(
|
||||
completion_response=transcript,
|
||||
model=f"{provider}/{model}",
|
||||
custom_llm_provider=provider,
|
||||
call_type="atranscription",
|
||||
)
|
||||
|
||||
assert cost_of_ten_seconds_without_usage("openai") > 0
|
||||
assert cost_of_ten_seconds_without_usage("azure") == 0.0, (
|
||||
"Azure lists only token prices for this model, https://azure.microsoft.com/en-us/pricing/details/azure-openai/ "
|
||||
"(per the 2026-10-28 review)"
|
||||
)
|
||||
|
||||
|
||||
def test_gpt_image_2_azure_cost_tracking():
|
||||
azure_image_generation_response: Final = {
|
||||
"created": 1758585600,
|
||||
|
|
@ -10409,6 +10522,301 @@ def test_responses_ws_logging_object_carries_the_completed_responses_id_tier_and
|
|||
assert ResponsesWebSocketTokenUsageProcessor.create_logging_object([{"type": "error", "error": {}}]).id is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ("openai", "azure"))
|
||||
@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini"))
|
||||
def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str):
|
||||
model_name: Final = f"azure/{model}" if provider == "azure" else model
|
||||
rates: Final = litellm.model_cost[model_name]
|
||||
events: Final[OpenAIRealtimeStreamList] = [
|
||||
{"type": "session.created", "session": {"model": model}},
|
||||
{
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"usage": {
|
||||
"input_tokens": 1000,
|
||||
"output_tokens": 300,
|
||||
"total_tokens": 1300,
|
||||
"input_token_details": {
|
||||
"text_tokens": 400,
|
||||
"audio_tokens": 400,
|
||||
"image_tokens": 200,
|
||||
"cached_tokens": 300,
|
||||
"cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50},
|
||||
},
|
||||
"output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100},
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events)
|
||||
actual: Final = handle_realtime_stream_cost_calculation(
|
||||
results=events,
|
||||
combined_usage_object=combined,
|
||||
custom_llm_provider=provider,
|
||||
litellm_model_name=model_name,
|
||||
)
|
||||
expected: Final = (
|
||||
300 * rates["input_cost_per_token"]
|
||||
+ 250 * rates["input_cost_per_audio_token"]
|
||||
+ 150 * rates["input_cost_per_image_token"]
|
||||
+ 100 * rates["cache_read_input_token_cost"]
|
||||
+ 150 * rates["cache_read_input_audio_token_cost"]
|
||||
+ 50 * rates["cache_read_input_image_token_cost"]
|
||||
+ 200 * rates["output_cost_per_token"]
|
||||
+ 100 * rates["output_cost_per_audio_token"]
|
||||
)
|
||||
|
||||
assert actual == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_realtime_translation_duration_cost(_local_model_cost_map):
|
||||
from litellm.cost_calculator import handle_realtime_translation_cost_calculation
|
||||
|
||||
model: Final = "gpt-realtime-translate"
|
||||
events: Final[OpenAIRealtimeStreamList] = [
|
||||
{"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 1.5}}
|
||||
]
|
||||
actual: Final = handle_realtime_translation_cost_calculation(
|
||||
results=events,
|
||||
custom_llm_provider="openai",
|
||||
litellm_model_name=model,
|
||||
)
|
||||
|
||||
assert actual == pytest.approx(2 * litellm.model_cost[model]["input_cost_per_second"])
|
||||
|
||||
|
||||
def test_realtime_translation_bills_session_audio_once_on_openai_and_azure(_local_model_cost_map) -> None:
|
||||
from litellm.cost_calculator import handle_realtime_translation_cost_calculation
|
||||
|
||||
one_minute_session: Final[OpenAIRealtimeStreamList] = [
|
||||
{"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}}
|
||||
]
|
||||
openai_cost: Final = handle_realtime_translation_cost_calculation(
|
||||
results=one_minute_session, custom_llm_provider="openai", litellm_model_name="gpt-realtime-translate"
|
||||
)
|
||||
azure_cost: Final = handle_realtime_translation_cost_calculation(
|
||||
results=one_minute_session, custom_llm_provider="azure", litellm_model_name="gpt-realtime-translate"
|
||||
)
|
||||
|
||||
assert openai_cost == pytest.approx(60 * litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"])
|
||||
assert openai_cost == pytest.approx(azure_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True])
|
||||
def test_realtime_translation_duration_cost_ignores_malformed_provider_usage(
|
||||
_local_model_cost_map, malformed_duration: float | int | bool
|
||||
) -> None:
|
||||
from litellm.cost_calculator import handle_realtime_translation_cost_calculation
|
||||
|
||||
model: Final = "translation-deployment-priced-both-directions"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "realtime",
|
||||
"input_cost_per_second": 0.001,
|
||||
"output_cost_per_second": 0.002,
|
||||
}
|
||||
}
|
||||
)
|
||||
input_invalid_events: Final[OpenAIRealtimeStreamList] = [
|
||||
{
|
||||
"type": "session.closed",
|
||||
"usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5},
|
||||
}
|
||||
]
|
||||
output_invalid_events: Final[OpenAIRealtimeStreamList] = [
|
||||
{
|
||||
"type": "session.closed",
|
||||
"usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration},
|
||||
}
|
||||
]
|
||||
|
||||
input_invalid_cost: Final = handle_realtime_translation_cost_calculation(
|
||||
results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model
|
||||
)
|
||||
output_invalid_cost: Final = handle_realtime_translation_cost_calculation(
|
||||
results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model
|
||||
)
|
||||
|
||||
assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"])
|
||||
assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"])
|
||||
|
||||
|
||||
def _translation_session_cost(
|
||||
deployment_id: str,
|
||||
session_model: str | None = "gpt-realtime-translate",
|
||||
model: str = "gpt-realtime-translate",
|
||||
base_model: str | None = None,
|
||||
) -> float:
|
||||
session_created: Final[OpenAIRealtimeStreamList] = (
|
||||
[{"type": "session.created", "session": {"model": session_model}}] if session_model is not None else []
|
||||
)
|
||||
one_minute_session: Final = LiteLLMRealtimeStreamLoggingObject(
|
||||
usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0),
|
||||
results=[
|
||||
*session_created,
|
||||
{"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}},
|
||||
],
|
||||
)
|
||||
return completion_cost(
|
||||
completion_response=one_minute_session,
|
||||
model=model,
|
||||
call_type=CallTypes.arealtime.value,
|
||||
custom_llm_provider="openai",
|
||||
custom_pricing=True,
|
||||
router_model_id=deployment_id,
|
||||
base_model=base_model,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("session_model", ["gpt-realtime-translate", None])
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_model,base_model",
|
||||
[("gpt-realtime-translate", None), ("my-translator", "gpt-realtime-translate")],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"deployment_id,deployment_rates,bills_public_input_rate",
|
||||
[
|
||||
("translate-output-priced", {"output_cost_per_second": 0.001}, True),
|
||||
("translate-free-input", {"input_cost_per_second": 0.0, "output_cost_per_second": 0.001}, False),
|
||||
],
|
||||
)
|
||||
def test_realtime_translation_deployment_per_second_rates_fall_back_to_public_rates_per_key(
|
||||
_local_model_cost_map: None,
|
||||
deployment_id: str,
|
||||
deployment_rates: Mapping[str, float],
|
||||
bills_public_input_rate: bool,
|
||||
deployment_model: str,
|
||||
base_model: str | None,
|
||||
session_model: str | None,
|
||||
) -> None:
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "translate",
|
||||
"litellm_params": {
|
||||
"model": f"openai/{deployment_model}",
|
||||
"api_key": "sk-test",
|
||||
**deployment_rates,
|
||||
},
|
||||
"model_info": {"id": deployment_id, **({"base_model": base_model} if base_model else {})},
|
||||
}
|
||||
]
|
||||
)
|
||||
public_input_rate: Final = litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"]
|
||||
|
||||
assert public_input_rate > 0
|
||||
assert _translation_session_cost(
|
||||
deployment_id, session_model=session_model, model=deployment_model, base_model=base_model
|
||||
) == pytest.approx(60 * 0.001 + (60 * public_input_rate if bills_public_input_rate else 0.0))
|
||||
|
||||
|
||||
def test_realtime_translation_custom_deployment_with_one_rate_bills_the_session_model_for_the_other(
|
||||
_local_model_cost_map: None,
|
||||
) -> None:
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "translate",
|
||||
"litellm_params": {
|
||||
"model": "openai/my-translator",
|
||||
"api_key": "sk-test",
|
||||
"output_cost_per_second": 0.001,
|
||||
},
|
||||
"model_info": {"id": "custom-translate-output-priced"},
|
||||
}
|
||||
]
|
||||
)
|
||||
public_input_rate: Final = litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"]
|
||||
|
||||
assert public_input_rate > 0
|
||||
assert _translation_session_cost(
|
||||
"custom-translate-output-priced", model="my-translator"
|
||||
) == pytest.approx(60 * 0.001 + 60 * public_input_rate)
|
||||
|
||||
|
||||
def test_realtime_translation_rate_ignores_a_session_model_the_deployment_does_not_run(
|
||||
_local_model_cost_map: None,
|
||||
) -> None:
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
"translation-session-echo": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "realtime",
|
||||
"input_cost_per_second": 0.5,
|
||||
}
|
||||
}
|
||||
)
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "translate",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-realtime-translate",
|
||||
"api_key": "sk-test",
|
||||
"output_cost_per_second": 0.001,
|
||||
},
|
||||
"model_info": {"id": "translate-output-priced"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert _translation_session_cost(
|
||||
"translate-output-priced", session_model="translation-session-echo"
|
||||
) == pytest.approx(_translation_session_cost("translate-output-priced", session_model=None))
|
||||
|
||||
|
||||
def test_realtime_translation_deployment_without_per_second_rates_bills_public_rates(_local_model_cost_map) -> None:
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "translate",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-realtime-translate",
|
||||
"api_key": "sk-test",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"model_info": {"id": "translate-token-priced"},
|
||||
}
|
||||
]
|
||||
)
|
||||
public_entry: Final = litellm.model_cost["gpt-realtime-translate"]
|
||||
public_cost: Final = 60 * (public_entry.get("input_cost_per_second") or 0.0) + 60 * (
|
||||
public_entry.get("output_cost_per_second") or 0.0
|
||||
)
|
||||
|
||||
assert public_cost > 0
|
||||
assert _translation_session_cost("translate-token-priced") == pytest.approx(public_cost)
|
||||
|
||||
|
||||
def test_realtime_translation_spend_usage_carries_the_billed_input_seconds(_local_model_cost_map) -> None:
|
||||
logging_obj: Final = Logging(
|
||||
model="gpt-realtime-translate",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type=CallTypes.arealtime.value,
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="translation-spend-usage",
|
||||
function_id="translation-spend-usage",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="gpt-realtime-translate", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai"
|
||||
)
|
||||
events: Final[OpenAIRealtimeStreamList] = [
|
||||
{"type": "session.closed", "usage": {"type": "duration", "input_seconds": 30.0, "output_seconds": 42.5}}
|
||||
]
|
||||
|
||||
logging_obj.success_handler(result=events, start_time=datetime.datetime.now(), end_time=datetime.datetime.now())
|
||||
|
||||
payload: Final = logging_obj.model_call_details["standard_logging_object"]
|
||||
assert payload["metadata"]["usage_object"]["seconds"] == pytest.approx(30.0)
|
||||
assert payload["response_cost"] == pytest.approx(
|
||||
30.0 * litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("profile_prefix", ("us", "global"))
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
|
|
|
|||
|
|
@ -6127,6 +6127,65 @@ async def test_groq_atranscription_honors_base_url_alias(respx_mock: respx.MockR
|
|||
assert response.text == "hello"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_body,expected_upload_measurements",
|
||||
[({"text": "hello", "duration": 2.5}, 0), ({"text": "hello"}, 1)],
|
||||
ids=["provider_reports_duration", "provider_omits_duration"],
|
||||
)
|
||||
def test_transcription_measures_the_upload_only_when_the_provider_omits_the_duration(
|
||||
respx_mock: respx.MockRouter, provider_body: Mapping[str, object], expected_upload_measurements: int
|
||||
):
|
||||
respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock(
|
||||
return_value=httpx.Response(200, json=provider_body)
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
litellm_main, "calculate_request_duration", wraps=litellm_main.calculate_request_duration
|
||||
) as measure_upload:
|
||||
response: Final = litellm.transcription(
|
||||
model="groq/whisper-large-v3", file=GROQ_WAV_FILE, base_url=GROQ_INTERNAL_BASE, api_key="fake-key"
|
||||
)
|
||||
|
||||
assert response.text == "hello"
|
||||
assert measure_upload.call_count == expected_upload_measurements
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", ("whisper-1", "gpt-4o-transcribe", "gpt-transcribe"))
|
||||
def test_openai_compatible_transcription_with_an_openai_model_name_reaches_the_server(
|
||||
respx_mock: respx.MockRouter, model_name: str
|
||||
):
|
||||
route: Final = respx_mock.post("https://vllm.internal/v1/audio/transcriptions").mock(
|
||||
return_value=httpx.Response(200, json={"text": "served by vllm"})
|
||||
)
|
||||
|
||||
response: Final = litellm.transcription(
|
||||
model=f"hosted_vllm/{model_name}",
|
||||
file=GROQ_WAV_FILE,
|
||||
api_base="https://vllm.internal/v1",
|
||||
api_key="fake-key",
|
||||
response_format="verbose_json",
|
||||
)
|
||||
|
||||
assert route.called
|
||||
assert response.text == "served by vllm"
|
||||
|
||||
|
||||
def test_openai_gpt_transcribe_still_rejects_non_json_response_formats(respx_mock: respx.MockRouter):
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock(
|
||||
return_value=httpx.Response(200, json={"text": "should not be sent"})
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"):
|
||||
litellm.transcription(
|
||||
model="openai/gpt-transcribe",
|
||||
file=GROQ_WAV_FILE,
|
||||
api_key="fake-key",
|
||||
response_format="srt",
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
def test_groq_speech_honors_base_url_alias(respx_mock: respx.MockRouter):
|
||||
route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/speech").mock(
|
||||
return_value=httpx.Response(200, content=b"mp3-bytes")
|
||||
|
|
|
|||
|
|
@ -838,6 +838,67 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
|
|||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini"))
|
||||
def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str):
|
||||
openai: Final = prices[model]
|
||||
azure: Final = prices[f"azure/{model}"]
|
||||
|
||||
assert openai["cache_read_input_image_token_cost"] > 0
|
||||
assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"]
|
||||
assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,mode",
|
||||
(
|
||||
("gpt-realtime-translate", "realtime"),
|
||||
("gpt-live-transcribe", "audio_transcription"),
|
||||
("gpt-transcribe", "audio_transcription"),
|
||||
),
|
||||
)
|
||||
def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str):
|
||||
openai: Final = prices[model]
|
||||
azure: Final = prices[f"azure/{model}"]
|
||||
|
||||
assert openai["mode"] == azure["mode"] == mode
|
||||
assert azure["supports_audio_input"] is True
|
||||
assert set(openai.get("supported_endpoints", ())) <= set(azure.get("supported_endpoints", ()))
|
||||
|
||||
|
||||
_DATED_SNAPSHOT: Final = re.compile(r"(?P<alias>.+)-\d{4}-\d{2}-\d{2}")
|
||||
|
||||
|
||||
def _azure_realtime_snapshots_with_an_alias(prices: Mapping[str, dict]) -> tuple[tuple[str, str], ...]:
|
||||
dated: Final = (
|
||||
(name, _DATED_SNAPSHOT.fullmatch(name))
|
||||
for name, entry in prices.items()
|
||||
if name.startswith("azure/") and "/v1/realtime" in entry.get("supported_endpoints", ())
|
||||
)
|
||||
return tuple((name, match["alias"]) for name, match in dated if match is not None and match["alias"] in prices)
|
||||
|
||||
|
||||
def _realtime_ga_only(entry: dict) -> object:
|
||||
return (entry.get("provider_specific_entry") or {}).get("realtime_ga_only")
|
||||
|
||||
|
||||
def test_dated_azure_realtime_models_select_the_same_protocol_as_their_alias(prices: dict):
|
||||
snapshots: Final = _azure_realtime_snapshots_with_an_alias(prices)
|
||||
disagreeing: Final = tuple(
|
||||
(snapshot, alias)
|
||||
for snapshot, alias in snapshots
|
||||
if _realtime_ga_only(prices[snapshot]) != _realtime_ga_only(prices[alias])
|
||||
)
|
||||
|
||||
assert snapshots
|
||||
assert disagreeing == ()
|
||||
|
||||
|
||||
def test_model_prices_backup_is_synchronized(prices: dict):
|
||||
backup: Final = json.loads(BACKUP_PRICES_PATH.read_text())
|
||||
|
||||
assert backup == prices
|
||||
|
||||
|
||||
def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]:
|
||||
prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
|
||||
return MappingProxyType(
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ from litellm.utils import (
|
|||
client,
|
||||
get_non_default_completion_params,
|
||||
get_optional_params_image_gen,
|
||||
get_optional_params_transcription,
|
||||
get_prompt_cache_min_tokens,
|
||||
get_supported_openai_params,
|
||||
get_token_count,
|
||||
|
|
@ -1247,6 +1248,9 @@ _COST_MAP_ROW_SCHEMA: Final = {
|
|||
"/v1/images/generations",
|
||||
"/v1/realtime",
|
||||
"/v1/realtime/transcription_sessions",
|
||||
"/v1/realtime/translations",
|
||||
"/v1/realtime/translations/client_secrets",
|
||||
"/v1/realtime/translations/calls",
|
||||
"/v1/images/variations",
|
||||
"/v1/images/edits",
|
||||
"/v1/batch",
|
||||
|
|
@ -10985,6 +10989,49 @@ def test_get_provider_audio_transcription_config_resolves_for_every_provider() -
|
|||
assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig)
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("custom_llm_provider", "model"),
|
||||
[
|
||||
("groq", "whisper-large-v3"),
|
||||
("deepgram", "nova-3"),
|
||||
("mistral", "voxtral-mini-latest"),
|
||||
("elevenlabs", "scribe_v1"),
|
||||
("fireworks_ai", "whisper-v3"),
|
||||
],
|
||||
)
|
||||
def test_openai_sdk_transcription_params_pass_through_to_other_providers(custom_llm_provider: str, model: str) -> None:
|
||||
hints: Final = {"stream": True, "languages": ["en"], "keywords": ["LiteLLM"]}
|
||||
|
||||
params: Final = get_optional_params_transcription(model=model, custom_llm_provider=custom_llm_provider, **hints)
|
||||
|
||||
forwarded: Final = {**params.get("extra_body", {}), **params}
|
||||
assert {key: forwarded.get(key) for key in hints} == hints
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_drop_params_keeps_openai_sdk_transcription_params_out_of_a_deepgram_request() -> None:
|
||||
route: Final = respx.post(url__startswith="https://api.deepgram.com/v1/listen").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"metadata": {"duration": 1.0}, "results": {"channels": [{"alternatives": [{"transcript": "hi"}]}]}},
|
||||
)
|
||||
)
|
||||
|
||||
litellm.transcription(
|
||||
model="deepgram/nova-3",
|
||||
file=("sample.wav", b"audio"),
|
||||
api_key="dg-test",
|
||||
punctuate=True,
|
||||
stream=True,
|
||||
languages=["en"],
|
||||
keywords=["LiteLLM"],
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert dict(route.calls.last.request.url.params) == {"model": "nova-3", "punctuate": "true"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "optional_params", "expected_response_format"),
|
||||
[
|
||||
|
|
|
|||
552
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
552
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -11098,7 +11098,7 @@ export interface paths {
|
|||
* WebSocket: realtime_websocket_endpoint
|
||||
* @description WebSocket connection endpoint
|
||||
*/
|
||||
get: operations["websocket_realtime_websocket_endpoint_get_3"];
|
||||
get: operations["websocket_realtime_websocket_endpoint_get_5"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
|
|
@ -11116,7 +11116,10 @@ export interface paths {
|
|||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Proxy Realtime Calls */
|
||||
/**
|
||||
* Proxy Realtime Calls
|
||||
* @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets
|
||||
*/
|
||||
post: operations["proxy_realtime_calls_openai_v1_realtime_calls_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
|
|
@ -11133,7 +11136,10 @@ export interface paths {
|
|||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Create Realtime Client Secret */
|
||||
/**
|
||||
* Create Realtime Client Secret
|
||||
* @description Create a short-lived client token for a Realtime conversation, transcription, or translation session
|
||||
*/
|
||||
post: operations["create_realtime_client_secret_openai_v1_realtime_client_secrets_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
|
|
@ -14075,7 +14081,7 @@ export interface paths {
|
|||
* WebSocket: realtime_websocket_endpoint
|
||||
* @description WebSocket connection endpoint
|
||||
*/
|
||||
get: operations["websocket_realtime_websocket_endpoint_get"];
|
||||
get: operations["websocket_realtime_websocket_endpoint_get_3"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
|
|
@ -14093,7 +14099,10 @@ export interface paths {
|
|||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Proxy Realtime Calls */
|
||||
/**
|
||||
* Proxy Realtime Calls
|
||||
* @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets
|
||||
*/
|
||||
post: operations["proxy_realtime_calls_realtime_calls_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
|
|
@ -14110,7 +14119,10 @@ export interface paths {
|
|||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Create Realtime Client Secret */
|
||||
/**
|
||||
* Create Realtime Client Secret
|
||||
* @description Create a short-lived client token for a Realtime conversation, transcription, or translation session
|
||||
*/
|
||||
post: operations["create_realtime_client_secret_realtime_client_secrets_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
|
|
@ -14142,6 +14154,66 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/realtime/translations": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* WebSocket: realtime_websocket_endpoint
|
||||
* @description WebSocket connection endpoint
|
||||
*/
|
||||
get: operations["websocket_realtime_websocket_endpoint_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/realtime/translations/calls": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Proxy Realtime Calls
|
||||
* @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets
|
||||
*/
|
||||
post: operations["proxy_realtime_calls_realtime_translations_calls_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/realtime/translations/client_secrets": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Create Realtime Client Secret
|
||||
* @description Create a short-lived client token for a Realtime conversation, transcription, or translation session
|
||||
*/
|
||||
post: operations["create_realtime_client_secret_realtime_translations_client_secrets_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/register": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -21893,7 +21965,7 @@ export interface paths {
|
|||
* WebSocket: realtime_websocket_endpoint
|
||||
* @description WebSocket connection endpoint
|
||||
*/
|
||||
get: operations["websocket_realtime_websocket_endpoint_get_2"];
|
||||
get: operations["websocket_realtime_websocket_endpoint_get_4"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
|
|
@ -21911,7 +21983,10 @@ export interface paths {
|
|||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Proxy Realtime Calls */
|
||||
/**
|
||||
* Proxy Realtime Calls
|
||||
* @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets
|
||||
*/
|
||||
post: operations["proxy_realtime_calls_v1_realtime_calls_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
|
|
@ -21928,7 +22003,10 @@ export interface paths {
|
|||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Create Realtime Client Secret */
|
||||
/**
|
||||
* Create Realtime Client Secret
|
||||
* @description Create a short-lived client token for a Realtime conversation, transcription, or translation session
|
||||
*/
|
||||
post: operations["create_realtime_client_secret_v1_realtime_client_secrets_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
|
|
@ -21960,6 +22038,66 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/realtime/translations": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* WebSocket: realtime_websocket_endpoint
|
||||
* @description WebSocket connection endpoint
|
||||
*/
|
||||
get: operations["websocket_realtime_websocket_endpoint_get_2"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/realtime/translations/calls": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Proxy Realtime Calls
|
||||
* @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets
|
||||
*/
|
||||
post: operations["proxy_realtime_calls_v1_realtime_translations_calls_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/realtime/translations/client_secrets": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Create Realtime Client Secret
|
||||
* @description Create a short-lived client token for a Realtime conversation, transcription, or translation session
|
||||
*/
|
||||
post: operations["create_realtime_client_secret_v1_realtime_translations_client_secrets_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/v1/rerank": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -28770,7 +28908,7 @@ export interface components {
|
|||
* CallTypes
|
||||
* @enum {string}
|
||||
*/
|
||||
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
|
||||
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
|
||||
/** CallbackDelete */
|
||||
CallbackDelete: {
|
||||
/** Callback Name */
|
||||
|
|
@ -48483,6 +48621,54 @@ export interface components {
|
|||
*/
|
||||
success_definition: string;
|
||||
};
|
||||
/** TranscriptionDetectedLanguage */
|
||||
TranscriptionDetectedLanguage: {
|
||||
/** Code */
|
||||
code: string;
|
||||
};
|
||||
/** TranscriptionResponse */
|
||||
TranscriptionResponse: {
|
||||
/** Languages */
|
||||
languages?: components["schemas"]["TranscriptionDetectedLanguage"][] | null;
|
||||
/** Text */
|
||||
text?: string | null;
|
||||
/** Usage */
|
||||
usage?: components["schemas"]["TranscriptionUsageDurationObject"] | components["schemas"]["TranscriptionUsageTokensObject"] | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** TranscriptionUsageDurationObject */
|
||||
TranscriptionUsageDurationObject: {
|
||||
/** Seconds */
|
||||
seconds: number;
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "duration";
|
||||
};
|
||||
/** TranscriptionUsageInputTokenDetailsObject */
|
||||
TranscriptionUsageInputTokenDetailsObject: {
|
||||
/** Audio Tokens */
|
||||
audio_tokens: number;
|
||||
/** Text Tokens */
|
||||
text_tokens: number;
|
||||
};
|
||||
/** TranscriptionUsageTokensObject */
|
||||
TranscriptionUsageTokensObject: {
|
||||
input_token_details?: components["schemas"]["TranscriptionUsageInputTokenDetailsObject"] | null;
|
||||
/** Input Tokens */
|
||||
input_tokens: number;
|
||||
/** Output Tokens */
|
||||
output_tokens: number;
|
||||
/** Total Tokens */
|
||||
total_tokens: number;
|
||||
/**
|
||||
* Type
|
||||
* @constant
|
||||
*/
|
||||
type: "tokens";
|
||||
};
|
||||
/** TransformRequestBody */
|
||||
TransformRequestBody: {
|
||||
call_type: components["schemas"]["CallTypes"];
|
||||
|
|
@ -53404,7 +53590,8 @@ export interface operations {
|
|||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
"application/json": components["schemas"]["TranscriptionResponse"];
|
||||
"text/event-stream": string;
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
|
|
@ -67716,7 +67903,7 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
websocket_realtime_websocket_endpoint_get_3: {
|
||||
websocket_realtime_websocket_endpoint_get_5: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
|
|
@ -71082,7 +71269,7 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
websocket_realtime_websocket_endpoint_get: {
|
||||
websocket_realtime_websocket_endpoint_get_3: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
|
|
@ -71160,6 +71347,173 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
websocket_realtime_websocket_endpoint_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description WebSocket Protocol Switched */
|
||||
101: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
proxy_realtime_calls_realtime_translations_calls_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/sdp": string;
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description WebRTC SDP answer */
|
||||
201: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"text/plain": string;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
create_realtime_client_secret_realtime_translations_client_secrets_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
/**
|
||||
* @example {
|
||||
* "model": "gpt-realtime-translate",
|
||||
* "session": {
|
||||
* "model": "gpt-realtime-translate",
|
||||
* "type": "translation"
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
"application/json": {
|
||||
expires_after?: {
|
||||
/**
|
||||
* Anchor
|
||||
* @default created_at
|
||||
*/
|
||||
anchor?: string | null;
|
||||
/** Seconds */
|
||||
seconds?: number | null;
|
||||
} | null;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
session?: ({
|
||||
audio?: ({
|
||||
input?: ({
|
||||
/** Format */
|
||||
format?: string | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Noise Reduction */
|
||||
noise_reduction?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
transcription?: ({
|
||||
/** Delay */
|
||||
delay?: string | null;
|
||||
/** Keywords */
|
||||
keywords?: string[] | null;
|
||||
/** Language */
|
||||
language?: string | null;
|
||||
/** Languages */
|
||||
languages?: string[] | null;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
/** Prompt */
|
||||
prompt?: string | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
/** Turn Detection */
|
||||
turn_detection?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
output?: ({
|
||||
/** Format */
|
||||
format?: string | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Language */
|
||||
language?: string | null;
|
||||
/** Voice */
|
||||
voice?: string | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
/** Include */
|
||||
include?: string[] | null;
|
||||
/** Instructions */
|
||||
instructions?: string | null;
|
||||
/** Max Output Tokens */
|
||||
max_output_tokens?: number | string | null;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
/** Output Modalities */
|
||||
output_modalities?: string[] | null;
|
||||
/** Prompt */
|
||||
prompt?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Tool Choice */
|
||||
tool_choice?: unknown | null;
|
||||
/** Tools */
|
||||
tools?: {
|
||||
[key: string]: unknown;
|
||||
}[] | null;
|
||||
/** Tracing */
|
||||
tracing?: unknown | null;
|
||||
/** Truncation */
|
||||
truncation?: unknown | null;
|
||||
/** Type */
|
||||
type?: string | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
};
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["RealtimeClientSecretResponse"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
register_client_register_post: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
@ -78613,7 +78967,8 @@ export interface operations {
|
|||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
"application/json": components["schemas"]["TranscriptionResponse"];
|
||||
"text/event-stream": string;
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
|
|
@ -82209,7 +82564,7 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
websocket_realtime_websocket_endpoint_get_2: {
|
||||
websocket_realtime_websocket_endpoint_get_4: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
|
|
@ -82287,6 +82642,173 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
websocket_realtime_websocket_endpoint_get_2: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description WebSocket Protocol Switched */
|
||||
101: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
proxy_realtime_calls_v1_realtime_translations_calls_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/sdp": string;
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description WebRTC SDP answer */
|
||||
201: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"text/plain": string;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
create_realtime_client_secret_v1_realtime_translations_client_secrets_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
/**
|
||||
* @example {
|
||||
* "model": "gpt-realtime-translate",
|
||||
* "session": {
|
||||
* "model": "gpt-realtime-translate",
|
||||
* "type": "translation"
|
||||
* }
|
||||
* }
|
||||
*/
|
||||
"application/json": {
|
||||
expires_after?: {
|
||||
/**
|
||||
* Anchor
|
||||
* @default created_at
|
||||
*/
|
||||
anchor?: string | null;
|
||||
/** Seconds */
|
||||
seconds?: number | null;
|
||||
} | null;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
session?: ({
|
||||
audio?: ({
|
||||
input?: ({
|
||||
/** Format */
|
||||
format?: string | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Noise Reduction */
|
||||
noise_reduction?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
transcription?: ({
|
||||
/** Delay */
|
||||
delay?: string | null;
|
||||
/** Keywords */
|
||||
keywords?: string[] | null;
|
||||
/** Language */
|
||||
language?: string | null;
|
||||
/** Languages */
|
||||
languages?: string[] | null;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
/** Prompt */
|
||||
prompt?: string | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
/** Turn Detection */
|
||||
turn_detection?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
output?: ({
|
||||
/** Format */
|
||||
format?: string | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Language */
|
||||
language?: string | null;
|
||||
/** Voice */
|
||||
voice?: string | {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
/** Include */
|
||||
include?: string[] | null;
|
||||
/** Instructions */
|
||||
instructions?: string | null;
|
||||
/** Max Output Tokens */
|
||||
max_output_tokens?: number | string | null;
|
||||
/** Model */
|
||||
model?: string | null;
|
||||
/** Output Modalities */
|
||||
output_modalities?: string[] | null;
|
||||
/** Prompt */
|
||||
prompt?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Tool Choice */
|
||||
tool_choice?: unknown | null;
|
||||
/** Tools */
|
||||
tools?: {
|
||||
[key: string]: unknown;
|
||||
}[] | null;
|
||||
/** Tracing */
|
||||
tracing?: unknown | null;
|
||||
/** Truncation */
|
||||
truncation?: unknown | null;
|
||||
/** Type */
|
||||
type?: string | null;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
}) | null;
|
||||
};
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["RealtimeClientSecretResponse"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
rerank_v1_rerank_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue