mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
feat(passthrough): add Deepgram streaming text-to-speech WebSocket route
/deepgram/v1/speak and /deepgram/speak relay Deepgram's streaming TTS socket the way /listen relays STT, with the same key auth on the requested voice, the server-side Deepgram key, and the same first-value-wins handling of a repeated model param A session is billed per character of the Speak text the client sent, at the row of the voice's Aura generation. Deepgram also speaks and bills text that was never flushed, so counting only flushed text would let a client get audio free, and counting on send can only over-bill text a Clear discards. A voice whose generation has no row is closed with 1008 before anything reaches Deepgram, and the spoken text never reaches the logs
This commit is contained in:
parent
aaecb51cf8
commit
7576d2e2aa
8 changed files with 443 additions and 14 deletions
|
|
@ -431,6 +431,7 @@ WEBSOCKET_CLIENT_TEXT_COUNT_KEY: Final = "litellm_websocket_client_text_count"
|
|||
DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
|
||||
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
|
||||
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
|
||||
DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
|
||||
|
||||
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
|
||||
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
import math
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL
|
||||
from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL, DEEPGRAM_SPEAK_DEFAULT_MODEL
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.utils import LlmProviders, ModelInfo
|
||||
|
|
@ -28,6 +30,7 @@ DEEPGRAM_LISTEN_ADDON_PRICING_PARAMS: Final = MappingProxyType(
|
|||
)
|
||||
_DISABLED_PARAM_VALUES: Final = frozenset({"", "false"})
|
||||
_SINGLE_VALUED_PARAMS: Final = frozenset({"model", "language"})
|
||||
_AURA_GENERATION: Final = re.compile(r"aura(?:-\d+)?(?=-)")
|
||||
|
||||
|
||||
class DeepgramException(BaseLLMException):
|
||||
|
|
@ -202,3 +205,39 @@ def deepgram_listen_transcript(websocket_messages: Sequence[Mapping[str, object]
|
|||
if frame.get("type") == "Results"
|
||||
if (transcript := _final_transcript(frame)) is not None
|
||||
)
|
||||
|
||||
|
||||
def deepgram_speak_model(upstream_url: str) -> str:
|
||||
models: Final = parse_qs(urlparse(upstream_url).query).get("model")
|
||||
return models[0] if models else DEEPGRAM_SPEAK_DEFAULT_MODEL
|
||||
|
||||
|
||||
def deepgram_speak_registry_key(upstream_url: str) -> str:
|
||||
"""Deepgram bills every voice of an Aura generation at one rate, so ``aura-2-thalia-en`` is priced at the
|
||||
``deepgram/aura-2`` row"""
|
||||
model: Final = deepgram_speak_model(upstream_url)
|
||||
generation: Final = _AURA_GENERATION.match(model)
|
||||
return f"{LlmProviders.DEEPGRAM.value}/{generation.group() if generation else model}"
|
||||
|
||||
|
||||
def deepgram_speak_character_rate(upstream_url: str) -> float | None:
|
||||
model_info: Final = _exact_registry_entry(deepgram_speak_registry_key(upstream_url))
|
||||
return None if model_info is None else model_info.get("input_cost_per_character")
|
||||
|
||||
|
||||
class _DeepgramSpeakMessage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
type: Literal["Speak"]
|
||||
text: str
|
||||
|
||||
|
||||
def deepgram_speak_characters(frame: str) -> int:
|
||||
"""Counts a Speak message's text as it is sent. Deepgram also speaks, and bills, unflushed text on its own (a long
|
||||
buffer, Close, a close frame), so counting only flushed text would let a client get audio free, while counting on
|
||||
send can only over-bill text that a Clear discards. Binary frames need no count: Deepgram closes the session on one.
|
||||
Never raises, since a traceback would carry the caller's text into the logs"""
|
||||
try:
|
||||
return len(_DeepgramSpeakMessage.model_validate_json(frame).text)
|
||||
except ValidationError:
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from litellm.constants import (
|
|||
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER,
|
||||
BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES,
|
||||
DEEPGRAM_LISTEN_DEFAULT_MODEL,
|
||||
DEEPGRAM_SPEAK_DEFAULT_MODEL,
|
||||
PASS_THROUGH_HEADER_PREFIX,
|
||||
TYPESAFE_DANGER_ZONE_HEADER_PREFIX,
|
||||
)
|
||||
|
|
@ -65,6 +66,9 @@ from litellm.llms.deepgram.common_utils import (
|
|||
deepgram_listen_is_priced,
|
||||
deepgram_listen_registry_key,
|
||||
deepgram_listen_requested_model,
|
||||
deepgram_speak_character_rate,
|
||||
deepgram_speak_characters,
|
||||
deepgram_speak_registry_key,
|
||||
deepgram_websocket_target,
|
||||
)
|
||||
from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
|
||||
|
|
@ -3457,6 +3461,7 @@ class _WebsocketRelay(Protocol):
|
|||
forward_headers: bool,
|
||||
endpoint: str,
|
||||
accept_websocket: bool,
|
||||
count_client_text: Callable[[str], int] | None = None,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
|
|
@ -3571,6 +3576,7 @@ class _DeepgramStream:
|
|||
path: str
|
||||
default_model: str
|
||||
refusal: Callable[[str], str | None]
|
||||
count_client_text: Callable[[str], int] | None = None
|
||||
|
||||
|
||||
def _deepgram_listen_refusal(target: str) -> str | None:
|
||||
|
|
@ -3587,6 +3593,20 @@ _DEEPGRAM_LISTEN: Final = _DeepgramStream(
|
|||
)
|
||||
|
||||
|
||||
def _deepgram_speak_refusal(target: str) -> str | None:
|
||||
if deepgram_speak_character_rate(target) is not None:
|
||||
return None
|
||||
return _DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_speak_registry_key(target))
|
||||
|
||||
|
||||
_DEEPGRAM_SPEAK: Final = _DeepgramStream(
|
||||
path="speak",
|
||||
default_model=DEEPGRAM_SPEAK_DEFAULT_MODEL,
|
||||
refusal=_deepgram_speak_refusal,
|
||||
count_client_text=deepgram_speak_characters,
|
||||
)
|
||||
|
||||
|
||||
async def _relay_deepgram_stream(
|
||||
websocket: WebSocket, user_api_key_dict: UserAPIKeyAuth, relay: _WebsocketRelay, stream: _DeepgramStream
|
||||
) -> None:
|
||||
|
|
@ -3618,6 +3638,7 @@ async def _relay_deepgram_stream(
|
|||
forward_headers=False,
|
||||
endpoint=websocket.url.path,
|
||||
accept_websocket=False,
|
||||
count_client_text=stream.count_client_text,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -3637,6 +3658,22 @@ async def deepgram_listen_websocket_route(
|
|||
await _relay_deepgram_stream(websocket, user_api_key_dict, relay, _DEEPGRAM_LISTEN)
|
||||
|
||||
|
||||
async def deepgram_speak_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
return await user_api_key_auth_websocket_for_model(
|
||||
websocket, model=httpx.QueryParams(websocket.url.query).get("model") or DEEPGRAM_SPEAK_DEFAULT_MODEL
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/deepgram/v1/speak")
|
||||
@router.websocket("/deepgram/speak")
|
||||
async def deepgram_speak_websocket_route(
|
||||
websocket: WebSocket,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_speak_user_api_key_auth)],
|
||||
relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
|
||||
) -> None:
|
||||
await _relay_deepgram_stream(websocket, user_api_key_dict, relay, _DEEPGRAM_SPEAK)
|
||||
|
||||
|
||||
_OPENAI_BETA_HEADER: Final = "openai-beta"
|
||||
_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"})
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
import json
|
||||
from typing import Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import litellm
|
||||
from litellm.constants import WEBSOCKET_CLIENT_TEXT_COUNT_KEY
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.deepgram.common_utils import deepgram_speak_character_rate, deepgram_speak_model
|
||||
from litellm.types.utils import StandardPassThroughResponseObject
|
||||
|
||||
DEEPGRAM_SPEAK_ROUTE_SUFFIX: Final = "/speak"
|
||||
|
||||
|
||||
class DeepgramSpeakPassthroughLoggingHandler:
|
||||
@staticmethod
|
||||
def is_deepgram_speak_route(url_route: str) -> bool:
|
||||
path: Final = urlparse(url_route).path
|
||||
return "/deepgram/" in path and path.endswith(DEEPGRAM_SPEAK_ROUTE_SUFFIX)
|
||||
|
||||
@staticmethod
|
||||
def deepgram_speak_passthrough_handler(
|
||||
logging_obj: LiteLLMLoggingObj, upstream_url: str
|
||||
) -> StandardPassThroughResponseObject:
|
||||
counted: Final = logging_obj.model_call_details.get(WEBSOCKET_CLIENT_TEXT_COUNT_KEY)
|
||||
characters: Final = counted if isinstance(counted, int) else 0
|
||||
rate: Final = deepgram_speak_character_rate(upstream_url)
|
||||
model: Final = deepgram_speak_model(upstream_url)
|
||||
logging_obj.model = model # rebind-ok: the spend logger reads model and cost off the shared logging object
|
||||
logging_obj.model_call_details["model"] = model # rebind-ok: same shared logging object
|
||||
logging_obj.model_call_details["custom_llm_provider"] = litellm.LlmProviders.DEEPGRAM.value # rebind-ok: same
|
||||
logging_obj.model_call_details["response_cost"] = None if rate is None else characters * rate # rebind-ok: same
|
||||
return StandardPassThroughResponseObject(response=json.dumps({"characters": characters}))
|
||||
|
|
@ -35,6 +35,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import (
|
|||
from .llm_provider_handlers.deepgram_listen_passthrough_logging_handler import (
|
||||
DeepgramListenPassthroughLoggingHandler,
|
||||
)
|
||||
from .llm_provider_handlers.deepgram_speak_passthrough_logging_handler import (
|
||||
DeepgramSpeakPassthroughLoggingHandler,
|
||||
)
|
||||
from .llm_provider_handlers.fal_ai_passthrough_logging_handler import (
|
||||
FalAIPassthroughLoggingHandler,
|
||||
)
|
||||
|
|
@ -187,7 +190,7 @@ class PassThroughEndpointLogging:
|
|||
|
||||
_handle_logging = handle_logging
|
||||
|
||||
def normalize_llm_passthrough_logging_payload(
|
||||
def normalize_llm_passthrough_logging_payload( # noqa: C901 # provider dispatch, one elif per provider route
|
||||
self,
|
||||
httpx_response: httpx.Response,
|
||||
response_body: dict | list[dict[str, object]] | None,
|
||||
|
|
@ -437,6 +440,12 @@ class PassThroughEndpointLogging:
|
|||
)
|
||||
standard_logging_response_object = deepgram_handler_result["result"] # rebind-ok: elif-chain
|
||||
kwargs = deepgram_handler_result["kwargs"] # rebind-ok: elif-chain contract
|
||||
elif DeepgramSpeakPassthroughLoggingHandler.is_deepgram_speak_route(url_route):
|
||||
standard_logging_response_object = ( # rebind-ok: elif-chain
|
||||
DeepgramSpeakPassthroughLoggingHandler.deepgram_speak_passthrough_handler(
|
||||
logging_obj=logging_obj, upstream_url=str(httpx_response.request.url)
|
||||
)
|
||||
)
|
||||
elif FalAIPassthroughLoggingHandler.is_fal_ai_route(url_route, custom_llm_provider):
|
||||
fal_ai_handler_result: Final = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler(
|
||||
response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}),
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@ from litellm.llms.deepgram.common_utils import (
|
|||
deepgram_listen_registry_key,
|
||||
deepgram_listen_requested_model,
|
||||
deepgram_listen_transcript,
|
||||
deepgram_speak_character_rate,
|
||||
deepgram_speak_characters,
|
||||
deepgram_speak_registry_key,
|
||||
deepgram_websocket_target,
|
||||
)
|
||||
|
||||
|
|
@ -324,3 +327,62 @@ def test_deepgram_listen_is_priced(
|
|||
)
|
||||
def test_deepgram_listen_addon_pricing_models(upstream_url: str, expected: tuple[str, ...]):
|
||||
assert deepgram_listen_addon_pricing_models(upstream_url) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected_target"),
|
||||
[
|
||||
pytest.param(
|
||||
"encoding=linear16",
|
||||
"wss://api.deepgram.com/v1/speak?encoding=linear16&model=aura-asteria-en",
|
||||
id="no model: Deepgram's own default, named so billing and upstream agree",
|
||||
),
|
||||
pytest.param(
|
||||
"model=aura-asteria-en&model=aura-2-thalia-en",
|
||||
"wss://api.deepgram.com/v1/speak?model=aura-asteria-en",
|
||||
id="second model dropped: Deepgram honours the last, auth and pricing read the first",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_deepgram_speak_websocket_target(query: str, expected_target: str):
|
||||
assert deepgram_websocket_target(None, "speak", query, "aura-asteria-en") == expected_target
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "registry_key", "priced"),
|
||||
[
|
||||
("aura-asteria-en", "deepgram/aura", True),
|
||||
("aura-2-thalia-en", "deepgram/aura-2", True),
|
||||
("aura-2-celeste-es", "deepgram/aura-2", True),
|
||||
("aura-99-x-en", "deepgram/aura-99", False),
|
||||
("nova-3", "deepgram/nova-3", False),
|
||||
(None, "deepgram/aura", True),
|
||||
],
|
||||
)
|
||||
def test_deepgram_speak_prices_each_voice_at_its_generation_row_only(
|
||||
model: str | None, registry_key: str, priced: bool
|
||||
):
|
||||
"""A voice is never billed at another generation's rate, and a listen model's row is not a speak price."""
|
||||
upstream_url = "wss://api.deepgram.com/v1/speak" + ("" if model is None else f"?model={model}")
|
||||
rate = deepgram_speak_character_rate(upstream_url)
|
||||
|
||||
assert deepgram_speak_registry_key(upstream_url) == registry_key
|
||||
assert rate == (litellm.model_cost[registry_key]["input_cost_per_character"] if priced else None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("frame", "characters"),
|
||||
[
|
||||
('{"type": "Speak", "text": "Hello world."}', 12),
|
||||
('{"type":"Speak","text":"\u00bfqu\u00e9 tal?"}', 9),
|
||||
('{"type": "Flush"}', 0),
|
||||
('{"type": "Speak"}', 0),
|
||||
('{"type": "Speak", "text": 12}', 0),
|
||||
('["Speak", "Hello"]', 0),
|
||||
("Hello", 0),
|
||||
("[" * 100_000, 0),
|
||||
],
|
||||
)
|
||||
def test_deepgram_speak_characters_counts_only_speak_text_and_never_raises(frame: str, characters: int):
|
||||
assert deepgram_speak_characters(frame) == characters
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
"""Deepgram ``/v1/speak`` WebSocket passthrough: per-character cost tracking."""
|
||||
|
||||
from datetime import datetime
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.constants import WEBSOCKET_CLIENT_TEXT_COUNT_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.deepgram_speak_passthrough_logging_handler import (
|
||||
DeepgramSpeakPassthroughLoggingHandler,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import PassthroughStandardLoggingPayload
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
AURA_2_URL: Final = "wss://api.deepgram.com/v1/speak?encoding=linear16&model=aura-2-thalia-en"
|
||||
|
||||
pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("url_route", "expected"),
|
||||
[
|
||||
("/deepgram/v1/speak", True),
|
||||
("/deepgram/speak", True),
|
||||
("/deepgram/v1/speak?model=aura-2-thalia-en", True),
|
||||
("/deepgram/v1/listen", False),
|
||||
("/openai/v1/audio/speak", False),
|
||||
("", False),
|
||||
],
|
||||
)
|
||||
def test_is_deepgram_speak_route(url_route: str, expected: bool):
|
||||
assert DeepgramSpeakPassthroughLoggingHandler.is_deepgram_speak_route(url_route) is expected
|
||||
|
||||
|
||||
class _CapturingLogger(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.payloads: list[StandardLoggingPayload] = []
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.payloads.append(kwargs["standard_logging_object"])
|
||||
|
||||
|
||||
async def _log_speak_session(monkeypatch: pytest.MonkeyPatch, characters: int | None) -> StandardLoggingPayload:
|
||||
"""Drives the shared passthrough success handler the way the WebSocket relay does at socket close and returns
|
||||
what a spend logger receives."""
|
||||
capturing_logger = _CapturingLogger()
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [capturing_logger])
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
logging_obj = LiteLLMLoggingObj(
|
||||
model="unknown",
|
||||
messages=[{"role": "user", "content": "WebSocket connection"}],
|
||||
stream=True,
|
||||
call_type="pass_through_endpoint",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id="call-dg-speak",
|
||||
function_id="websocket_passthrough",
|
||||
)
|
||||
litellm_params: Final = {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "team-tts"}}
|
||||
logging_obj.update_environment_variables(
|
||||
model="unknown",
|
||||
user="unknown",
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
call_type="pass_through_endpoint",
|
||||
)
|
||||
if characters is not None:
|
||||
logging_obj.model_call_details[WEBSOCKET_CLIENT_TEXT_COUNT_KEY] = characters
|
||||
|
||||
await PassThroughEndpointLogging().pass_through_async_success_handler(
|
||||
httpx_response=SimpleNamespace(
|
||||
status_code=200,
|
||||
text="WebSocket connection successful",
|
||||
headers={},
|
||||
request=SimpleNamespace(method="WEBSOCKET", url=AURA_2_URL),
|
||||
),
|
||||
response_body=[{"type": "Metadata", "model_name": "aura-2-thalia-en"}, {"type": "Flushed", "sequence_id": 0}],
|
||||
logging_obj=logging_obj,
|
||||
url_route="/deepgram/v1/speak",
|
||||
result="websocket_connection_successful",
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
passthrough_logging_payload=PassthroughStandardLoggingPayload(
|
||||
url=AURA_2_URL, request_body={}, request_method="WEBSOCKET", cost_per_request=None
|
||||
),
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
assert len(capturing_logger.payloads) == 1
|
||||
return capturing_logger.payloads[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_handler_bills_the_counted_characters_at_the_voice_generation_rate(monkeypatch):
|
||||
payload = await _log_speak_session(monkeypatch, characters=47)
|
||||
|
||||
assert payload["model"] == "aura-2-thalia-en"
|
||||
assert payload["custom_llm_provider"] == "deepgram"
|
||||
assert payload["response_cost"] == pytest.approx(
|
||||
47 * litellm.model_cost["deepgram/aura-2"]["input_cost_per_character"]
|
||||
)
|
||||
assert payload["response"] == {"response": '{"characters": 47}'}
|
||||
assert payload["metadata"]["user_api_key_team_id"] == "team-tts"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_handler_bills_nothing_for_a_session_that_spoke_no_text(monkeypatch):
|
||||
payload = await _log_speak_session(monkeypatch, characters=None)
|
||||
|
||||
assert payload["model"] == "aura-2-thalia-en"
|
||||
assert payload["response_cost"] == 0
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
"""Deepgram ``/v1/listen`` passthrough WebSocket route: registration, auth, credential injection, target URL."""
|
||||
"""Deepgram ``/v1/listen`` and ``/v1/speak`` passthrough WebSocket routes: registration, auth, credential injection,
|
||||
target URL."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final
|
||||
|
|
@ -15,12 +16,15 @@ from starlette.websockets import WebSocketDisconnect
|
|||
|
||||
import litellm
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import DEEPGRAM_SPEAK_DEFAULT_MODEL
|
||||
from litellm.llms.deepgram.common_utils import deepgram_speak_characters
|
||||
from litellm.proxy._lazy_features import LAZY_FEATURES
|
||||
from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import cache_key_object
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
_websocket_relay,
|
||||
deepgram_listen_websocket_route,
|
||||
deepgram_speak_websocket_route,
|
||||
router,
|
||||
)
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
|
@ -30,6 +34,11 @@ GET_CREDENTIALS: Final = (
|
|||
)
|
||||
USER_API_KEY_AUTH: Final = "litellm.proxy.auth.user_api_key_auth.user_api_key_auth"
|
||||
LISTEN_PATHS: Final = ("/deepgram/v1/listen", "/deepgram/listen")
|
||||
SPEAK_PATHS: Final = ("/deepgram/v1/speak", "/deepgram/speak")
|
||||
ROUTES: Final = (
|
||||
pytest.param(deepgram_listen_websocket_route, "/deepgram/v1/listen", id="listen"),
|
||||
pytest.param(deepgram_speak_websocket_route, "/deepgram/v1/speak", id="speak"),
|
||||
)
|
||||
NOVA_2_STREAMING_KEY: Final = "deepgram/streaming/nova-2"
|
||||
|
||||
pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map")
|
||||
|
|
@ -62,6 +71,7 @@ class _RelayCall:
|
|||
forward_headers: bool
|
||||
endpoint: str
|
||||
accept_websocket: bool
|
||||
count_client_text: Callable[[str], int] | None = None
|
||||
|
||||
|
||||
class _FakeRelay:
|
||||
|
|
@ -78,6 +88,7 @@ class _FakeRelay:
|
|||
forward_headers: bool,
|
||||
endpoint: str,
|
||||
accept_websocket: bool,
|
||||
count_client_text: Callable[[str], int] | None = None,
|
||||
) -> None:
|
||||
self.calls.append(
|
||||
_RelayCall(
|
||||
|
|
@ -87,13 +98,18 @@ class _FakeRelay:
|
|||
forward_headers=forward_headers,
|
||||
endpoint=endpoint,
|
||||
accept_websocket=accept_websocket,
|
||||
count_client_text=count_client_text,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _serve(websocket: _FakeWebSocket, user_api_key_dict: UserAPIKeyAuth | None = None) -> _FakeRelay:
|
||||
async def _serve(
|
||||
websocket: _FakeWebSocket,
|
||||
user_api_key_dict: UserAPIKeyAuth | None = None,
|
||||
route: Callable[..., Awaitable[None]] = deepgram_listen_websocket_route,
|
||||
) -> _FakeRelay:
|
||||
relay = _FakeRelay()
|
||||
await deepgram_listen_websocket_route(
|
||||
await route(
|
||||
websocket=websocket,
|
||||
user_api_key_dict=user_api_key_dict or UserAPIKeyAuth(),
|
||||
relay=relay,
|
||||
|
|
@ -208,11 +224,12 @@ async def test_deepgram_listen_ignores_caller_supplied_api_base(monkeypatch):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deepgram_listen_closes_cleanly_when_provider_credentials_missing():
|
||||
websocket = _FakeWebSocket("/deepgram/v1/listen", "model=nova-3")
|
||||
@pytest.mark.parametrize(("route", "path"), ROUTES)
|
||||
async def test_deepgram_ws_routes_close_cleanly_when_provider_credentials_missing(route, path):
|
||||
websocket = _FakeWebSocket(path, "")
|
||||
|
||||
with patch(GET_CREDENTIALS, return_value=None):
|
||||
relay = await _serve(websocket)
|
||||
relay = await _serve(websocket, route=route)
|
||||
|
||||
assert websocket.closed is not None
|
||||
assert websocket.closed[0] == 1011
|
||||
|
|
@ -299,13 +316,14 @@ def _app_with_relay(relay: _FakeRelay) -> FastAPI:
|
|||
return app
|
||||
|
||||
|
||||
def test_deepgram_listen_rejects_connections_without_a_litellm_key():
|
||||
@pytest.mark.parametrize("path", (LISTEN_PATHS[0], SPEAK_PATHS[0]))
|
||||
def test_deepgram_ws_routes_reject_connections_without_a_litellm_key(path):
|
||||
relay = _FakeRelay()
|
||||
client = TestClient(_app_with_relay(relay))
|
||||
|
||||
with patch(GET_CREDENTIALS, return_value="dg-provider-key") as get_credentials:
|
||||
with pytest.raises(WebSocketDisconnect) as disconnect:
|
||||
with client.websocket_connect("/deepgram/v1/listen?model=nova-3"):
|
||||
with client.websocket_connect(path):
|
||||
pass
|
||||
|
||||
assert disconnect.value.code == 1008
|
||||
|
|
@ -451,7 +469,8 @@ def test_deepgram_listen_strips_a_second_model_that_would_outrank_the_authorized
|
|||
assert [call.target for call in relay.calls] == ["wss://api.deepgram.com/v1/listen?model=nova-2&language=en"]
|
||||
|
||||
|
||||
def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm_key(monkeypatch):
|
||||
@pytest.mark.parametrize("path", (LISTEN_PATHS[0], SPEAK_PATHS[0]))
|
||||
def test_deepgram_ws_routes_echo_the_browser_subprotocol_that_carries_the_litellm_key(path, monkeypatch):
|
||||
"""Browsers cannot set headers, so they send the key as a subprotocol and abort the handshake unless the
|
||||
server echoes that subprotocol back; the key itself must still stay off the upstream connection."""
|
||||
monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)
|
||||
|
|
@ -463,7 +482,7 @@ def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm
|
|||
patch(USER_API_KEY_AUTH, new=AsyncMock(return_value=UserAPIKeyAuth(api_key="hashed"))),
|
||||
):
|
||||
with client.websocket_connect(
|
||||
"/deepgram/v1/listen?model=nova-3",
|
||||
path,
|
||||
subprotocols=["openai-insecure-api-key.sk-litellm-virtual"],
|
||||
) as connection:
|
||||
assert connection.accepted_subprotocol == "openai-insecure-api-key.sk-litellm-virtual"
|
||||
|
|
@ -472,3 +491,114 @@ def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm
|
|||
MappingProxyType({"Authorization": "Token dg-provider-key"})
|
||||
]
|
||||
assert [call.forward_headers for call in relay.calls] == [False]
|
||||
|
||||
|
||||
def test_deepgram_speak_websocket_routes_registered():
|
||||
ws_paths = {route.path for route in router.routes if isinstance(route, WebSocketRoute)}
|
||||
assert set(SPEAK_PATHS) <= ws_paths
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", SPEAK_PATHS)
|
||||
async def test_deepgram_speak_relays_with_provider_auth_and_counts_the_spoken_characters(path, monkeypatch):
|
||||
monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)
|
||||
websocket = _FakeWebSocket(path, "encoding=linear16&sample_rate=24000")
|
||||
caller = UserAPIKeyAuth(api_key="sk-litellm-virtual", team_id="team-tts")
|
||||
relay = _FakeRelay()
|
||||
|
||||
with patch(GET_CREDENTIALS, return_value="dg-provider-key"):
|
||||
await deepgram_speak_websocket_route(websocket=websocket, user_api_key_dict=caller, relay=relay)
|
||||
|
||||
assert relay.calls == [
|
||||
_RelayCall(
|
||||
target=f"wss://api.deepgram.com/v1/speak?encoding=linear16&sample_rate=24000&model={DEEPGRAM_SPEAK_DEFAULT_MODEL}",
|
||||
custom_headers=MappingProxyType({"Authorization": "Token dg-provider-key"}),
|
||||
user_api_key_dict=caller,
|
||||
forward_headers=False,
|
||||
endpoint=path,
|
||||
accept_websocket=False,
|
||||
count_client_text=deepgram_speak_characters,
|
||||
)
|
||||
]
|
||||
assert websocket.closed is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deepgram_speak_honours_server_configured_api_base(monkeypatch):
|
||||
monkeypatch.setenv("DEEPGRAM_API_BASE", "http://localhost:8080/v1")
|
||||
websocket = _FakeWebSocket("/deepgram/v1/speak", "model=aura-2-thalia-en")
|
||||
|
||||
with patch(GET_CREDENTIALS, return_value="dg-provider-key"):
|
||||
relay = await _serve(websocket, route=deepgram_speak_websocket_route)
|
||||
|
||||
assert [call.target for call in relay.calls] == ["ws://localhost:8080/v1/speak?model=aura-2-thalia-en"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("query", "missing_key"),
|
||||
[
|
||||
pytest.param("model=aura-99-x-en", "deepgram/aura-99", id="generation without a row"),
|
||||
pytest.param("model=nova-3", "deepgram/nova-3", id="listen model has no per-character rate"),
|
||||
],
|
||||
)
|
||||
async def test_deepgram_speak_refuses_voices_it_cannot_price(query, missing_key, monkeypatch):
|
||||
monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)
|
||||
websocket = _FakeWebSocket("/deepgram/v1/speak", query)
|
||||
|
||||
with patch(GET_CREDENTIALS, return_value="dg-provider-key"):
|
||||
relay = await _serve(websocket, route=deepgram_speak_websocket_route)
|
||||
|
||||
assert relay.calls == []
|
||||
assert websocket.closed == (
|
||||
1008,
|
||||
f"No streaming price for '{missing_key}': add it to the model cost map to enable it",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected_target"),
|
||||
[
|
||||
pytest.param("model=aura-2-thalia-en", "wss://api.deepgram.com/v1/speak?model=aura-2-thalia-en", id="allowed"),
|
||||
pytest.param("model=aura-asteria-en", None, id="denied voice named"),
|
||||
pytest.param("", None, id="voice omitted, default denied"),
|
||||
pytest.param(
|
||||
"model=aura-2-thalia-en&model=aura-asteria-en",
|
||||
"wss://api.deepgram.com/v1/speak?model=aura-2-thalia-en",
|
||||
id="second voice stripped",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_deepgram_speak_authorizes_the_voice_it_will_actually_send_upstream(query, expected_target, monkeypatch):
|
||||
monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "max_budget", 0.0)
|
||||
cache = asyncio.run(_cache_restricted_key("sk-only-thalia", ["aura-2-thalia-en"]))
|
||||
relay = _FakeRelay()
|
||||
client = TestClient(_app_with_relay(relay))
|
||||
|
||||
with (
|
||||
patch(GET_CREDENTIALS, return_value="dg-provider-key"),
|
||||
patch.multiple( # test-quality-ok: the real key auth path reads these proxy_server globals and has no injection seam
|
||||
"litellm.proxy.proxy_server",
|
||||
master_key="sk-master",
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=cache,
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
),
|
||||
):
|
||||
if expected_target is not None:
|
||||
with client.websocket_connect(
|
||||
f"/deepgram/v1/speak?{query}", headers={"Authorization": "Bearer sk-only-thalia"}
|
||||
):
|
||||
pass
|
||||
assert [call.target for call in relay.calls] == [expected_target]
|
||||
return
|
||||
with pytest.raises(WebSocketDisconnect) as disconnect:
|
||||
with client.websocket_connect(
|
||||
f"/deepgram/v1/speak?{query}", headers={"Authorization": "Bearer sk-only-thalia"}
|
||||
):
|
||||
pass
|
||||
|
||||
assert disconnect.value.code == 1008
|
||||
assert relay.calls == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue