diff --git a/litellm/constants.py b/litellm/constants.py index bdf5921b1f2..9856d436a19 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -431,6 +431,7 @@ WEBSOCKET_CLIENT_TEXT_COUNT_KEY: Final = "litellm_websocket_client_text_count" DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1" NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1" DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3" +DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en" BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message" BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed" diff --git a/litellm/llms/deepgram/common_utils.py b/litellm/llms/deepgram/common_utils.py index f57f12f1239..1bce86c203c 100644 --- a/litellm/llms/deepgram/common_utils.py +++ b/litellm/llms/deepgram/common_utils.py @@ -1,13 +1,15 @@ import math +import re from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final +from typing import Final, Literal from urllib.parse import parse_qs, urlparse import httpx +from pydantic import BaseModel, ConfigDict, ValidationError import litellm -from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL +from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL, DEEPGRAM_SPEAK_DEFAULT_MODEL from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import LlmProviders, ModelInfo @@ -28,6 +30,7 @@ DEEPGRAM_LISTEN_ADDON_PRICING_PARAMS: Final = MappingProxyType( ) _DISABLED_PARAM_VALUES: Final = frozenset({"", "false"}) _SINGLE_VALUED_PARAMS: Final = frozenset({"model", "language"}) +_AURA_GENERATION: Final = re.compile(r"aura(?:-\d+)?(?=-)") class DeepgramException(BaseLLMException): @@ -202,3 +205,39 @@ def deepgram_listen_transcript(websocket_messages: Sequence[Mapping[str, object] if frame.get("type") == "Results" if (transcript := _final_transcript(frame)) is not None ) + + +def deepgram_speak_model(upstream_url: str) -> str: + models: Final = parse_qs(urlparse(upstream_url).query).get("model") + return models[0] if models else DEEPGRAM_SPEAK_DEFAULT_MODEL + + +def deepgram_speak_registry_key(upstream_url: str) -> str: + """Deepgram bills every voice of an Aura generation at one rate, so ``aura-2-thalia-en`` is priced at the + ``deepgram/aura-2`` row""" + model: Final = deepgram_speak_model(upstream_url) + generation: Final = _AURA_GENERATION.match(model) + return f"{LlmProviders.DEEPGRAM.value}/{generation.group() if generation else model}" + + +def deepgram_speak_character_rate(upstream_url: str) -> float | None: + model_info: Final = _exact_registry_entry(deepgram_speak_registry_key(upstream_url)) + return None if model_info is None else model_info.get("input_cost_per_character") + + +class _DeepgramSpeakMessage(BaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["Speak"] + text: str + + +def deepgram_speak_characters(frame: str) -> int: + """Counts a Speak message's text as it is sent. Deepgram also speaks, and bills, unflushed text on its own (a long + buffer, Close, a close frame), so counting only flushed text would let a client get audio free, while counting on + send can only over-bill text that a Clear discards. Binary frames need no count: Deepgram closes the session on one. + Never raises, since a traceback would carry the caller's text into the logs""" + try: + return len(_DeepgramSpeakMessage.model_validate_json(frame).text) + except ValidationError: + return 0 diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 14096513303..baadae8a02d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -46,6 +46,7 @@ from litellm.constants import ( AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, DEEPGRAM_LISTEN_DEFAULT_MODEL, + DEEPGRAM_SPEAK_DEFAULT_MODEL, PASS_THROUGH_HEADER_PREFIX, TYPESAFE_DANGER_ZONE_HEADER_PREFIX, ) @@ -65,6 +66,9 @@ from litellm.llms.deepgram.common_utils import ( deepgram_listen_is_priced, deepgram_listen_registry_key, deepgram_listen_requested_model, + deepgram_speak_character_rate, + deepgram_speak_characters, + deepgram_speak_registry_key, deepgram_websocket_target, ) from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base @@ -3457,6 +3461,7 @@ class _WebsocketRelay(Protocol): forward_headers: bool, endpoint: str, accept_websocket: bool, + count_client_text: Callable[[str], int] | None = None, ) -> None: ... @@ -3571,6 +3576,7 @@ class _DeepgramStream: path: str default_model: str refusal: Callable[[str], str | None] + count_client_text: Callable[[str], int] | None = None def _deepgram_listen_refusal(target: str) -> str | None: @@ -3587,6 +3593,20 @@ _DEEPGRAM_LISTEN: Final = _DeepgramStream( ) +def _deepgram_speak_refusal(target: str) -> str | None: + if deepgram_speak_character_rate(target) is not None: + return None + return _DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_speak_registry_key(target)) + + +_DEEPGRAM_SPEAK: Final = _DeepgramStream( + path="speak", + default_model=DEEPGRAM_SPEAK_DEFAULT_MODEL, + refusal=_deepgram_speak_refusal, + count_client_text=deepgram_speak_characters, +) + + async def _relay_deepgram_stream( websocket: WebSocket, user_api_key_dict: UserAPIKeyAuth, relay: _WebsocketRelay, stream: _DeepgramStream ) -> None: @@ -3618,6 +3638,7 @@ async def _relay_deepgram_stream( forward_headers=False, endpoint=websocket.url.path, accept_websocket=False, + count_client_text=stream.count_client_text, ) @@ -3637,6 +3658,22 @@ async def deepgram_listen_websocket_route( await _relay_deepgram_stream(websocket, user_api_key_dict, relay, _DEEPGRAM_LISTEN) +async def deepgram_speak_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth: + return await user_api_key_auth_websocket_for_model( + websocket, model=httpx.QueryParams(websocket.url.query).get("model") or DEEPGRAM_SPEAK_DEFAULT_MODEL + ) + + +@router.websocket("/deepgram/v1/speak") +@router.websocket("/deepgram/speak") +async def deepgram_speak_websocket_route( + websocket: WebSocket, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_speak_user_api_key_auth)], + relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)], +) -> None: + await _relay_deepgram_stream(websocket, user_api_key_dict, relay, _DEEPGRAM_SPEAK) + + _OPENAI_BETA_HEADER: Final = "openai-beta" _CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"}) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/deepgram_speak_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/deepgram_speak_passthrough_logging_handler.py new file mode 100644 index 00000000000..ec179f0d5b7 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/deepgram_speak_passthrough_logging_handler.py @@ -0,0 +1,32 @@ +import json +from typing import Final +from urllib.parse import urlparse + +import litellm +from litellm.constants import WEBSOCKET_CLIENT_TEXT_COUNT_KEY +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.deepgram.common_utils import deepgram_speak_character_rate, deepgram_speak_model +from litellm.types.utils import StandardPassThroughResponseObject + +DEEPGRAM_SPEAK_ROUTE_SUFFIX: Final = "/speak" + + +class DeepgramSpeakPassthroughLoggingHandler: + @staticmethod + def is_deepgram_speak_route(url_route: str) -> bool: + path: Final = urlparse(url_route).path + return "/deepgram/" in path and path.endswith(DEEPGRAM_SPEAK_ROUTE_SUFFIX) + + @staticmethod + def deepgram_speak_passthrough_handler( + logging_obj: LiteLLMLoggingObj, upstream_url: str + ) -> StandardPassThroughResponseObject: + counted: Final = logging_obj.model_call_details.get(WEBSOCKET_CLIENT_TEXT_COUNT_KEY) + characters: Final = counted if isinstance(counted, int) else 0 + rate: Final = deepgram_speak_character_rate(upstream_url) + model: Final = deepgram_speak_model(upstream_url) + logging_obj.model = model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = litellm.LlmProviders.DEEPGRAM.value # rebind-ok: same + logging_obj.model_call_details["response_cost"] = None if rate is None else characters * rate # rebind-ok: same + return StandardPassThroughResponseObject(response=json.dumps({"characters": characters})) diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 1f4e7b6df95..65549486434 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -35,6 +35,9 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import ( from .llm_provider_handlers.deepgram_listen_passthrough_logging_handler import ( DeepgramListenPassthroughLoggingHandler, ) +from .llm_provider_handlers.deepgram_speak_passthrough_logging_handler import ( + DeepgramSpeakPassthroughLoggingHandler, +) from .llm_provider_handlers.fal_ai_passthrough_logging_handler import ( FalAIPassthroughLoggingHandler, ) @@ -187,7 +190,7 @@ class PassThroughEndpointLogging: _handle_logging = handle_logging - def normalize_llm_passthrough_logging_payload( + def normalize_llm_passthrough_logging_payload( # noqa: C901 # provider dispatch, one elif per provider route self, httpx_response: httpx.Response, response_body: dict | list[dict[str, object]] | None, @@ -437,6 +440,12 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = deepgram_handler_result["result"] # rebind-ok: elif-chain kwargs = deepgram_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif DeepgramSpeakPassthroughLoggingHandler.is_deepgram_speak_route(url_route): + standard_logging_response_object = ( # rebind-ok: elif-chain + DeepgramSpeakPassthroughLoggingHandler.deepgram_speak_passthrough_handler( + logging_obj=logging_obj, upstream_url=str(httpx_response.request.url) + ) + ) elif FalAIPassthroughLoggingHandler.is_fal_ai_route(url_route, custom_llm_provider): fal_ai_handler_result: Final = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}), diff --git a/tests/unit/llms/deepgram/test_deepgram_common_utils.py b/tests/unit/llms/deepgram/test_deepgram_common_utils.py index a9e2efa90b7..44d4c21e637 100644 --- a/tests/unit/llms/deepgram/test_deepgram_common_utils.py +++ b/tests/unit/llms/deepgram/test_deepgram_common_utils.py @@ -17,6 +17,9 @@ from litellm.llms.deepgram.common_utils import ( deepgram_listen_registry_key, deepgram_listen_requested_model, deepgram_listen_transcript, + deepgram_speak_character_rate, + deepgram_speak_characters, + deepgram_speak_registry_key, deepgram_websocket_target, ) @@ -324,3 +327,62 @@ def test_deepgram_listen_is_priced( ) def test_deepgram_listen_addon_pricing_models(upstream_url: str, expected: tuple[str, ...]): assert deepgram_listen_addon_pricing_models(upstream_url) == expected + + +@pytest.mark.parametrize( + ("query", "expected_target"), + [ + pytest.param( + "encoding=linear16", + "wss://api.deepgram.com/v1/speak?encoding=linear16&model=aura-asteria-en", + id="no model: Deepgram's own default, named so billing and upstream agree", + ), + pytest.param( + "model=aura-asteria-en&model=aura-2-thalia-en", + "wss://api.deepgram.com/v1/speak?model=aura-asteria-en", + id="second model dropped: Deepgram honours the last, auth and pricing read the first", + ), + ], +) +def test_deepgram_speak_websocket_target(query: str, expected_target: str): + assert deepgram_websocket_target(None, "speak", query, "aura-asteria-en") == expected_target + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "registry_key", "priced"), + [ + ("aura-asteria-en", "deepgram/aura", True), + ("aura-2-thalia-en", "deepgram/aura-2", True), + ("aura-2-celeste-es", "deepgram/aura-2", True), + ("aura-99-x-en", "deepgram/aura-99", False), + ("nova-3", "deepgram/nova-3", False), + (None, "deepgram/aura", True), + ], +) +def test_deepgram_speak_prices_each_voice_at_its_generation_row_only( + model: str | None, registry_key: str, priced: bool +): + """A voice is never billed at another generation's rate, and a listen model's row is not a speak price.""" + upstream_url = "wss://api.deepgram.com/v1/speak" + ("" if model is None else f"?model={model}") + rate = deepgram_speak_character_rate(upstream_url) + + assert deepgram_speak_registry_key(upstream_url) == registry_key + assert rate == (litellm.model_cost[registry_key]["input_cost_per_character"] if priced else None) + + +@pytest.mark.parametrize( + ("frame", "characters"), + [ + ('{"type": "Speak", "text": "Hello world."}', 12), + ('{"type":"Speak","text":"\u00bfqu\u00e9 tal?"}', 9), + ('{"type": "Flush"}', 0), + ('{"type": "Speak"}', 0), + ('{"type": "Speak", "text": 12}', 0), + ('["Speak", "Hello"]', 0), + ("Hello", 0), + ("[" * 100_000, 0), + ], +) +def test_deepgram_speak_characters_counts_only_speak_text_and_never_raises(frame: str, characters: int): + assert deepgram_speak_characters(frame) == characters diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_deepgram_speak_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_deepgram_speak_passthrough_logging_handler.py new file mode 100644 index 00000000000..7edf1df7165 --- /dev/null +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_deepgram_speak_passthrough_logging_handler.py @@ -0,0 +1,119 @@ +"""Deepgram ``/v1/speak`` WebSocket passthrough: per-character cost tracking.""" + +from datetime import datetime +from types import SimpleNamespace +from typing import Final + +import pytest + +import litellm +from litellm.constants import WEBSOCKET_CLIENT_TEXT_COUNT_KEY +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.deepgram_speak_passthrough_logging_handler import ( + DeepgramSpeakPassthroughLoggingHandler, +) +from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging +from litellm.types.passthrough_endpoints.pass_through_endpoints import PassthroughStandardLoggingPayload +from litellm.types.utils import StandardLoggingPayload + +AURA_2_URL: Final = "wss://api.deepgram.com/v1/speak?encoding=linear16&model=aura-2-thalia-en" + +pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map") + + +@pytest.mark.parametrize( + ("url_route", "expected"), + [ + ("/deepgram/v1/speak", True), + ("/deepgram/speak", True), + ("/deepgram/v1/speak?model=aura-2-thalia-en", True), + ("/deepgram/v1/listen", False), + ("/openai/v1/audio/speak", False), + ("", False), + ], +) +def test_is_deepgram_speak_route(url_route: str, expected: bool): + assert DeepgramSpeakPassthroughLoggingHandler.is_deepgram_speak_route(url_route) is expected + + +class _CapturingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payloads: list[StandardLoggingPayload] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.payloads.append(kwargs["standard_logging_object"]) + + +async def _log_speak_session(monkeypatch: pytest.MonkeyPatch, characters: int | None) -> StandardLoggingPayload: + """Drives the shared passthrough success handler the way the WebSocket relay does at socket close and returns + what a spend logger receives.""" + capturing_logger = _CapturingLogger() + monkeypatch.setattr(litellm, "_async_success_callback", [capturing_logger]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "callbacks", []) + logging_obj = LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "WebSocket connection"}], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="call-dg-speak", + function_id="websocket_passthrough", + ) + litellm_params: Final = {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "team-tts"}} + logging_obj.update_environment_variables( + model="unknown", + user="unknown", + optional_params={}, + litellm_params=litellm_params, + call_type="pass_through_endpoint", + ) + if characters is not None: + logging_obj.model_call_details[WEBSOCKET_CLIENT_TEXT_COUNT_KEY] = characters + + await PassThroughEndpointLogging().pass_through_async_success_handler( + httpx_response=SimpleNamespace( + status_code=200, + text="WebSocket connection successful", + headers={}, + request=SimpleNamespace(method="WEBSOCKET", url=AURA_2_URL), + ), + response_body=[{"type": "Metadata", "model_name": "aura-2-thalia-en"}, {"type": "Flushed", "sequence_id": 0}], + logging_obj=logging_obj, + url_route="/deepgram/v1/speak", + result="websocket_connection_successful", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + passthrough_logging_payload=PassthroughStandardLoggingPayload( + url=AURA_2_URL, request_body={}, request_method="WEBSOCKET", cost_per_request=None + ), + litellm_params=litellm_params, + ) + + assert len(capturing_logger.payloads) == 1 + return capturing_logger.payloads[0] + + +@pytest.mark.asyncio +async def test_success_handler_bills_the_counted_characters_at_the_voice_generation_rate(monkeypatch): + payload = await _log_speak_session(monkeypatch, characters=47) + + assert payload["model"] == "aura-2-thalia-en" + assert payload["custom_llm_provider"] == "deepgram" + assert payload["response_cost"] == pytest.approx( + 47 * litellm.model_cost["deepgram/aura-2"]["input_cost_per_character"] + ) + assert payload["response"] == {"response": '{"characters": 47}'} + assert payload["metadata"]["user_api_key_team_id"] == "team-tts" + + +@pytest.mark.asyncio +async def test_success_handler_bills_nothing_for_a_session_that_spoke_no_text(monkeypatch): + payload = await _log_speak_session(monkeypatch, characters=None) + + assert payload["model"] == "aura-2-thalia-en" + assert payload["response_cost"] == 0 diff --git a/tests/unit/proxy/pass_through_endpoints/test_deepgram_ws_passthrough_routes.py b/tests/unit/proxy/pass_through_endpoints/test_deepgram_ws_passthrough_routes.py index 404ed0f5800..e3902ac0272 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_deepgram_ws_passthrough_routes.py +++ b/tests/unit/proxy/pass_through_endpoints/test_deepgram_ws_passthrough_routes.py @@ -1,7 +1,8 @@ -"""Deepgram ``/v1/listen`` passthrough WebSocket route: registration, auth, credential injection, target URL.""" +"""Deepgram ``/v1/listen`` and ``/v1/speak`` passthrough WebSocket routes: registration, auth, credential injection, +target URL.""" import asyncio -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from types import MappingProxyType, SimpleNamespace from typing import Final @@ -15,12 +16,15 @@ from starlette.websockets import WebSocketDisconnect import litellm from litellm.caching.dual_cache import DualCache +from litellm.constants import DEEPGRAM_SPEAK_DEFAULT_MODEL +from litellm.llms.deepgram.common_utils import deepgram_speak_characters from litellm.proxy._lazy_features import LAZY_FEATURES from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import cache_key_object from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _websocket_relay, deepgram_listen_websocket_route, + deepgram_speak_websocket_route, router, ) from litellm.proxy.utils import hash_token @@ -30,6 +34,11 @@ GET_CREDENTIALS: Final = ( ) USER_API_KEY_AUTH: Final = "litellm.proxy.auth.user_api_key_auth.user_api_key_auth" LISTEN_PATHS: Final = ("/deepgram/v1/listen", "/deepgram/listen") +SPEAK_PATHS: Final = ("/deepgram/v1/speak", "/deepgram/speak") +ROUTES: Final = ( + pytest.param(deepgram_listen_websocket_route, "/deepgram/v1/listen", id="listen"), + pytest.param(deepgram_speak_websocket_route, "/deepgram/v1/speak", id="speak"), +) NOVA_2_STREAMING_KEY: Final = "deepgram/streaming/nova-2" pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map") @@ -62,6 +71,7 @@ class _RelayCall: forward_headers: bool endpoint: str accept_websocket: bool + count_client_text: Callable[[str], int] | None = None class _FakeRelay: @@ -78,6 +88,7 @@ class _FakeRelay: forward_headers: bool, endpoint: str, accept_websocket: bool, + count_client_text: Callable[[str], int] | None = None, ) -> None: self.calls.append( _RelayCall( @@ -87,13 +98,18 @@ class _FakeRelay: forward_headers=forward_headers, endpoint=endpoint, accept_websocket=accept_websocket, + count_client_text=count_client_text, ) ) -async def _serve(websocket: _FakeWebSocket, user_api_key_dict: UserAPIKeyAuth | None = None) -> _FakeRelay: +async def _serve( + websocket: _FakeWebSocket, + user_api_key_dict: UserAPIKeyAuth | None = None, + route: Callable[..., Awaitable[None]] = deepgram_listen_websocket_route, +) -> _FakeRelay: relay = _FakeRelay() - await deepgram_listen_websocket_route( + await route( websocket=websocket, user_api_key_dict=user_api_key_dict or UserAPIKeyAuth(), relay=relay, @@ -208,11 +224,12 @@ async def test_deepgram_listen_ignores_caller_supplied_api_base(monkeypatch): @pytest.mark.asyncio -async def test_deepgram_listen_closes_cleanly_when_provider_credentials_missing(): - websocket = _FakeWebSocket("/deepgram/v1/listen", "model=nova-3") +@pytest.mark.parametrize(("route", "path"), ROUTES) +async def test_deepgram_ws_routes_close_cleanly_when_provider_credentials_missing(route, path): + websocket = _FakeWebSocket(path, "") with patch(GET_CREDENTIALS, return_value=None): - relay = await _serve(websocket) + relay = await _serve(websocket, route=route) assert websocket.closed is not None assert websocket.closed[0] == 1011 @@ -299,13 +316,14 @@ def _app_with_relay(relay: _FakeRelay) -> FastAPI: return app -def test_deepgram_listen_rejects_connections_without_a_litellm_key(): +@pytest.mark.parametrize("path", (LISTEN_PATHS[0], SPEAK_PATHS[0])) +def test_deepgram_ws_routes_reject_connections_without_a_litellm_key(path): relay = _FakeRelay() client = TestClient(_app_with_relay(relay)) with patch(GET_CREDENTIALS, return_value="dg-provider-key") as get_credentials: with pytest.raises(WebSocketDisconnect) as disconnect: - with client.websocket_connect("/deepgram/v1/listen?model=nova-3"): + with client.websocket_connect(path): pass assert disconnect.value.code == 1008 @@ -451,7 +469,8 @@ def test_deepgram_listen_strips_a_second_model_that_would_outrank_the_authorized assert [call.target for call in relay.calls] == ["wss://api.deepgram.com/v1/listen?model=nova-2&language=en"] -def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm_key(monkeypatch): +@pytest.mark.parametrize("path", (LISTEN_PATHS[0], SPEAK_PATHS[0])) +def test_deepgram_ws_routes_echo_the_browser_subprotocol_that_carries_the_litellm_key(path, monkeypatch): """Browsers cannot set headers, so they send the key as a subprotocol and abort the handshake unless the server echoes that subprotocol back; the key itself must still stay off the upstream connection.""" monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False) @@ -463,7 +482,7 @@ def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm patch(USER_API_KEY_AUTH, new=AsyncMock(return_value=UserAPIKeyAuth(api_key="hashed"))), ): with client.websocket_connect( - "/deepgram/v1/listen?model=nova-3", + path, subprotocols=["openai-insecure-api-key.sk-litellm-virtual"], ) as connection: assert connection.accepted_subprotocol == "openai-insecure-api-key.sk-litellm-virtual" @@ -472,3 +491,114 @@ def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm MappingProxyType({"Authorization": "Token dg-provider-key"}) ] assert [call.forward_headers for call in relay.calls] == [False] + + +def test_deepgram_speak_websocket_routes_registered(): + ws_paths = {route.path for route in router.routes if isinstance(route, WebSocketRoute)} + assert set(SPEAK_PATHS) <= ws_paths + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", SPEAK_PATHS) +async def test_deepgram_speak_relays_with_provider_auth_and_counts_the_spoken_characters(path, monkeypatch): + monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False) + websocket = _FakeWebSocket(path, "encoding=linear16&sample_rate=24000") + caller = UserAPIKeyAuth(api_key="sk-litellm-virtual", team_id="team-tts") + relay = _FakeRelay() + + with patch(GET_CREDENTIALS, return_value="dg-provider-key"): + await deepgram_speak_websocket_route(websocket=websocket, user_api_key_dict=caller, relay=relay) + + assert relay.calls == [ + _RelayCall( + target=f"wss://api.deepgram.com/v1/speak?encoding=linear16&sample_rate=24000&model={DEEPGRAM_SPEAK_DEFAULT_MODEL}", + custom_headers=MappingProxyType({"Authorization": "Token dg-provider-key"}), + user_api_key_dict=caller, + forward_headers=False, + endpoint=path, + accept_websocket=False, + count_client_text=deepgram_speak_characters, + ) + ] + assert websocket.closed is None + + +@pytest.mark.asyncio +async def test_deepgram_speak_honours_server_configured_api_base(monkeypatch): + monkeypatch.setenv("DEEPGRAM_API_BASE", "http://localhost:8080/v1") + websocket = _FakeWebSocket("/deepgram/v1/speak", "model=aura-2-thalia-en") + + with patch(GET_CREDENTIALS, return_value="dg-provider-key"): + relay = await _serve(websocket, route=deepgram_speak_websocket_route) + + assert [call.target for call in relay.calls] == ["ws://localhost:8080/v1/speak?model=aura-2-thalia-en"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("query", "missing_key"), + [ + pytest.param("model=aura-99-x-en", "deepgram/aura-99", id="generation without a row"), + pytest.param("model=nova-3", "deepgram/nova-3", id="listen model has no per-character rate"), + ], +) +async def test_deepgram_speak_refuses_voices_it_cannot_price(query, missing_key, monkeypatch): + monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False) + websocket = _FakeWebSocket("/deepgram/v1/speak", query) + + with patch(GET_CREDENTIALS, return_value="dg-provider-key"): + relay = await _serve(websocket, route=deepgram_speak_websocket_route) + + assert relay.calls == [] + assert websocket.closed == ( + 1008, + f"No streaming price for '{missing_key}': add it to the model cost map to enable it", + ) + + +@pytest.mark.parametrize( + ("query", "expected_target"), + [ + pytest.param("model=aura-2-thalia-en", "wss://api.deepgram.com/v1/speak?model=aura-2-thalia-en", id="allowed"), + pytest.param("model=aura-asteria-en", None, id="denied voice named"), + pytest.param("", None, id="voice omitted, default denied"), + pytest.param( + "model=aura-2-thalia-en&model=aura-asteria-en", + "wss://api.deepgram.com/v1/speak?model=aura-2-thalia-en", + id="second voice stripped", + ), + ], +) +def test_deepgram_speak_authorizes_the_voice_it_will_actually_send_upstream(query, expected_target, monkeypatch): + monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False) + monkeypatch.setattr(litellm, "max_budget", 0.0) + cache = asyncio.run(_cache_restricted_key("sk-only-thalia", ["aura-2-thalia-en"])) + relay = _FakeRelay() + client = TestClient(_app_with_relay(relay)) + + with ( + patch(GET_CREDENTIALS, return_value="dg-provider-key"), + patch.multiple( # test-quality-ok: the real key auth path reads these proxy_server globals and has no injection seam + "litellm.proxy.proxy_server", + master_key="sk-master", + prisma_client=MagicMock(), + user_api_key_cache=cache, + llm_model_list=None, + llm_router=None, + ), + ): + if expected_target is not None: + with client.websocket_connect( + f"/deepgram/v1/speak?{query}", headers={"Authorization": "Bearer sk-only-thalia"} + ): + pass + assert [call.target for call in relay.calls] == [expected_target] + return + with pytest.raises(WebSocketDisconnect) as disconnect: + with client.websocket_connect( + f"/deepgram/v1/speak?{query}", headers={"Authorization": "Bearer sk-only-thalia"} + ): + pass + + assert disconnect.value.code == 1008 + assert relay.calls == []