mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
fix(deepgram): authorize the effective model and price /listen sessions at streaming rates
Key auth on the Deepgram WebSocket route now sees the same model the upstream target will carry, so a key restricted to other models can no longer reach nova-3 by leaving model out of the query. user_api_key_auth_websocket keeps its signature and delegates to user_api_key_auth_websocket_for_model, which the Deepgram route calls with deepgram_listen_requested_model Sessions are priced from new deepgram/streaming/* registry rows (nova-3, nova-3-multilingual for language=multi) plus per-minute add-on rows for redact, keyterm, detect_entities and diarize, all read from Deepgram's pricing page on 2026-09-17. Models without a streaming row fall back to their pre-recorded row as before Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin
parent
db560ca652
commit
11ec157d71
9 changed files with 481 additions and 22 deletions
|
|
@ -11,12 +11,29 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
|||
|
||||
_WEBSOCKET_SCHEMES: Final = MappingProxyType({"https": "wss", "http": "ws", "wss": "wss", "ws": "ws"})
|
||||
DEEPGRAM_LISTEN_CALLBACK_PARAMS: Final = frozenset({"callback", "callback_method"})
|
||||
DEEPGRAM_LISTEN_STREAMING_PRICING_PREFIX: Final = "streaming/"
|
||||
DEEPGRAM_LISTEN_MULTILINGUAL_LANGUAGE: Final = "multi"
|
||||
DEEPGRAM_LISTEN_MULTILINGUAL_PRICING_SUFFIX: Final = "-multilingual"
|
||||
DEEPGRAM_LISTEN_ADDON_PRICING_PARAMS: Final = MappingProxyType(
|
||||
{
|
||||
"redact": "redact",
|
||||
"keyterm": "keyterm",
|
||||
"detect_entities": "detect_entities",
|
||||
"diarize": "diarize",
|
||||
"diarize_model": "diarize",
|
||||
}
|
||||
)
|
||||
_DISABLED_PARAM_VALUES: Final = frozenset({"", "false"})
|
||||
|
||||
|
||||
class DeepgramException(BaseLLMException):
|
||||
pass
|
||||
|
||||
|
||||
def deepgram_listen_requested_model(query_string: str) -> str:
|
||||
return httpx.QueryParams(query_string).get("model") or DEEPGRAM_LISTEN_DEFAULT_MODEL
|
||||
|
||||
|
||||
def deepgram_listen_websocket_target(api_base: str | None, query_string: str) -> str:
|
||||
listen_url: Final = httpx.URL(f"{(api_base or DEEPGRAM_DEFAULT_API_BASE).rstrip('/')}/listen")
|
||||
websocket_url: Final = listen_url.copy_with(scheme=_WEBSOCKET_SCHEMES.get(listen_url.scheme, listen_url.scheme))
|
||||
|
|
@ -36,6 +53,38 @@ def deepgram_listen_model(upstream_url: str) -> str:
|
|||
return models[0] if models else DEEPGRAM_LISTEN_DEFAULT_MODEL
|
||||
|
||||
|
||||
def _param_enabled(values: Sequence[str]) -> bool:
|
||||
return any(value.strip().lower() not in _DISABLED_PARAM_VALUES for value in values)
|
||||
|
||||
|
||||
def deepgram_listen_base_pricing_models(upstream_url: str) -> tuple[str, ...]:
|
||||
"""Registry keys to try, in order, for the per-second base rate of a streaming session: the streaming entry for
|
||||
the language mode Deepgram bills (multilingual when ``language=multi``), then the plain streaming entry, then
|
||||
the pre-recorded entry for models that have no streaming price of their own."""
|
||||
model: Final = deepgram_listen_model(upstream_url)
|
||||
params: Final = parse_qs(urlparse(upstream_url).query)
|
||||
streaming: Final = f"{DEEPGRAM_LISTEN_STREAMING_PRICING_PREFIX}{model}"
|
||||
multilingual: Final = params.get("language", ("",))[-1].strip().lower() == DEEPGRAM_LISTEN_MULTILINGUAL_LANGUAGE
|
||||
return (
|
||||
(f"{streaming}{DEEPGRAM_LISTEN_MULTILINGUAL_PRICING_SUFFIX}", streaming, model)
|
||||
if multilingual
|
||||
else (streaming, model)
|
||||
)
|
||||
|
||||
|
||||
def deepgram_listen_addon_pricing_models(upstream_url: str) -> tuple[str, ...]:
|
||||
params: Final = parse_qs(urlparse(upstream_url).query)
|
||||
return tuple(
|
||||
sorted(
|
||||
frozenset(
|
||||
f"{DEEPGRAM_LISTEN_STREAMING_PRICING_PREFIX}{addon}"
|
||||
for param, addon in DEEPGRAM_LISTEN_ADDON_PRICING_PARAMS.items()
|
||||
if _param_enabled(params.get(param, ()))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _channel_count(value: object) -> int | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -20736,6 +20736,96 @@
|
|||
"/v1/audio/transcriptions"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3": {
|
||||
"input_cost_per_second": 8e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
|
||||
"note": "Nova-3 monolingual streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0048
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3-multilingual": {
|
||||
"input_cost_per_second": 9.667e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
|
||||
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0058
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/redact": {
|
||||
"input_cost_per_second": 3.333e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0020/60 seconds = $0.00003333 per second",
|
||||
"note": "Redaction add-on (redact query param), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.002
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/keyterm": {
|
||||
"input_cost_per_second": 2.167e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0013/60 seconds = $0.00002167 per second",
|
||||
"note": "Keyterm Prompting add-on (keyterm query param), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0013
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/detect_entities": {
|
||||
"input_cost_per_second": 2.833e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0017/60 seconds = $0.00002833 per second",
|
||||
"note": "Entity Detection add-on (detect_entities query param), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0017
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/diarize": {
|
||||
"input_cost_per_second": 3.333e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0020/60 seconds = $0.00003333 per second",
|
||||
"note": "Speaker Diarization add-on (diarize / diarize_model query params), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.002
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/whisper": {
|
||||
"input_cost_per_second": 0.0001,
|
||||
"litellm_provider": "deepgram",
|
||||
|
|
|
|||
|
|
@ -629,9 +629,11 @@ def _apply_budget_limits_to_end_user_params(
|
|||
verbose_proxy_logger.debug("Applied budget limits to end user %s", end_user_id)
|
||||
|
||||
|
||||
async def user_api_key_auth_websocket(websocket: WebSocket):
|
||||
# Accept the WebSocket connection
|
||||
async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model"))
|
||||
|
||||
|
||||
async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str | None) -> UserAPIKeyAuth:
|
||||
ws_scope: Final = websocket.scope or {}
|
||||
scope_headers: Final = list(ws_scope.get("headers") or [])
|
||||
# ``get_request_route`` falls back to ``request.url.path`` when
|
||||
|
|
@ -651,10 +653,6 @@ async def user_api_key_auth_websocket(websocket: WebSocket):
|
|||
|
||||
request._url = websocket.url
|
||||
|
||||
query_params: Final = websocket.query_params
|
||||
|
||||
model: Final = query_params.get("model")
|
||||
|
||||
async def return_body():
|
||||
return _realtime_request_body(model)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from litellm.llms.azure.passthrough.transformation import foreign_azure_deployme
|
|||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.deepgram.common_utils import (
|
||||
deepgram_listen_callback_params,
|
||||
deepgram_listen_requested_model,
|
||||
deepgram_listen_websocket_target,
|
||||
)
|
||||
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
|
||||
|
|
@ -52,6 +53,7 @@ from litellm.proxy.auth.user_api_key_auth import (
|
|||
is_no_auth_dev_mode,
|
||||
user_api_key_auth,
|
||||
user_api_key_auth_websocket,
|
||||
user_api_key_auth_websocket_for_model,
|
||||
)
|
||||
from litellm.proxy.common_request_processing import open_sse_before_first_byte
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
|
|
@ -2700,11 +2702,17 @@ _DEEPGRAM_WS_MISSING_KEY_REASON: Final = (
|
|||
_DEEPGRAM_WS_CALLBACK_REASON: Final = "Deepgram callback delivery is not supported through the proxy: remove {params}"
|
||||
|
||||
|
||||
async def deepgram_listen_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
return await user_api_key_auth_websocket_for_model(
|
||||
websocket, model=deepgram_listen_requested_model(websocket.url.query)
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/deepgram/v1/listen")
|
||||
@router.websocket("/deepgram/listen")
|
||||
async def deepgram_listen_websocket_route(
|
||||
websocket: WebSocket,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)],
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_listen_user_api_key_auth)],
|
||||
relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
|
||||
) -> None:
|
||||
deepgram_api_key: Final = passthrough_endpoint_router.get_credentials(
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.deepgram.common_utils import (
|
||||
deepgram_listen_addon_pricing_models,
|
||||
deepgram_listen_audio_seconds,
|
||||
deepgram_listen_base_pricing_models,
|
||||
deepgram_listen_channel_count,
|
||||
deepgram_listen_model,
|
||||
deepgram_listen_transcript,
|
||||
|
|
@ -18,19 +20,39 @@ from litellm.types.utils import TranscriptionResponse
|
|||
DEEPGRAM_LISTEN_ROUTE_SUFFIX: Final = "/listen"
|
||||
|
||||
|
||||
def _audio_cost(response: TranscriptionResponse, model: str) -> float | None:
|
||||
def _registry_cost(response: TranscriptionResponse, pricing_model: str) -> float | None:
|
||||
try:
|
||||
return litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
model=pricing_model,
|
||||
custom_llm_provider=litellm.LlmProviders.DEEPGRAM.value,
|
||||
call_type="transcription",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # an unpriced model must not lose the spend row, only its cost
|
||||
verbose_proxy_logger.warning("Deepgram listen passthrough: no pricing for model '%s': %s", model, e)
|
||||
except Exception as e: # noqa: BLE001 # an unpriced entry must not lose the spend row, only its cost
|
||||
verbose_proxy_logger.debug("Deepgram listen passthrough: no registry price for '%s': %s", pricing_model, e)
|
||||
return None
|
||||
|
||||
|
||||
def _audio_cost(response: TranscriptionResponse, upstream_url: str) -> float | None:
|
||||
base_cost: Final = next(
|
||||
(
|
||||
cost
|
||||
for pricing_model in deepgram_listen_base_pricing_models(upstream_url)
|
||||
if (cost := _registry_cost(response, pricing_model)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
if base_cost is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Deepgram listen passthrough: no pricing for model '%s'", deepgram_listen_model(upstream_url)
|
||||
)
|
||||
return None
|
||||
addon_costs: Final = tuple(
|
||||
_registry_cost(response, pricing_model) for pricing_model in deepgram_listen_addon_pricing_models(upstream_url)
|
||||
)
|
||||
return base_cost + sum(cost for cost in addon_costs if cost is not None)
|
||||
|
||||
|
||||
class DeepgramListenPassthroughLoggingHandler:
|
||||
@staticmethod
|
||||
def is_deepgram_listen_route(url_route: str) -> bool:
|
||||
|
|
@ -50,7 +72,7 @@ class DeepgramListenPassthroughLoggingHandler:
|
|||
billed_seconds: Final = audio_seconds * channels
|
||||
response: Final = TranscriptionResponse(text=deepgram_listen_transcript(websocket_messages))
|
||||
response._hidden_params["audio_transcription_duration"] = billed_seconds # pyright: ignore[reportPrivateUsage] # the cost calculator reads the billed duration off the response's hidden params
|
||||
response_cost: Final = _audio_cost(response, model)
|
||||
response_cost: Final = _audio_cost(response, upstream_url)
|
||||
response._hidden_params["response_cost"] = response_cost # pyright: ignore[reportPrivateUsage] # the logger reads a precomputed cost off the response's hidden params
|
||||
|
||||
provider: Final = litellm.LlmProviders.DEEPGRAM.value
|
||||
|
|
|
|||
|
|
@ -20736,6 +20736,96 @@
|
|||
"/v1/audio/transcriptions"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3": {
|
||||
"input_cost_per_second": 8e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0048/60 seconds = $0.00008000 per second",
|
||||
"note": "Nova-3 monolingual streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0048
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/nova-3-multilingual": {
|
||||
"input_cost_per_second": 9.667e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0058/60 seconds = $0.00009667 per second",
|
||||
"note": "Nova-3 multilingual (language=multi) streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0058
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/redact": {
|
||||
"input_cost_per_second": 3.333e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0020/60 seconds = $0.00003333 per second",
|
||||
"note": "Redaction add-on (redact query param), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.002
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/keyterm": {
|
||||
"input_cost_per_second": 2.167e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0013/60 seconds = $0.00002167 per second",
|
||||
"note": "Keyterm Prompting add-on (keyterm query param), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0013
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/detect_entities": {
|
||||
"input_cost_per_second": 2.833e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0017/60 seconds = $0.00002833 per second",
|
||||
"note": "Entity Detection add-on (detect_entities query param), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.0017
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/streaming/diarize": {
|
||||
"input_cost_per_second": 3.333e-05,
|
||||
"litellm_provider": "deepgram",
|
||||
"metadata": {
|
||||
"calculation": "$0.0020/60 seconds = $0.00003333 per second",
|
||||
"note": "Speaker Diarization add-on (diarize / diarize_model query params), streaming, pay as you go",
|
||||
"original_pricing_per_minute": 0.002
|
||||
},
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_second": 0.0,
|
||||
"source": "https://deepgram.com/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/listen"
|
||||
]
|
||||
},
|
||||
"deepgram/whisper": {
|
||||
"input_cost_per_second": 0.0001,
|
||||
"litellm_provider": "deepgram",
|
||||
|
|
|
|||
|
|
@ -6,10 +6,13 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.llms.deepgram.common_utils import (
|
||||
deepgram_listen_addon_pricing_models,
|
||||
deepgram_listen_audio_seconds,
|
||||
deepgram_listen_base_pricing_models,
|
||||
deepgram_listen_callback_params,
|
||||
deepgram_listen_channel_count,
|
||||
deepgram_listen_model,
|
||||
deepgram_listen_requested_model,
|
||||
deepgram_listen_transcript,
|
||||
deepgram_listen_websocket_target,
|
||||
)
|
||||
|
|
@ -195,3 +198,68 @@ def test_deepgram_listen_transcript_joins_final_results_only():
|
|||
)
|
||||
def test_deepgram_listen_model_comes_from_the_upstream_query(upstream_url: str, expected_model: str):
|
||||
assert deepgram_listen_model(upstream_url) == expected_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_string",
|
||||
["model=nova-2&language=en", "language=en", "model=&language=en", "", "model=nova-3-medical"],
|
||||
)
|
||||
def test_requested_model_is_the_model_the_upstream_target_will_carry(query_string: str):
|
||||
"""Authorization runs against ``deepgram_listen_requested_model``; the upstream URL is built separately, so the
|
||||
two must always agree or a key could be authorized for one model and reach another."""
|
||||
target: Final = deepgram_listen_websocket_target(None, query_string)
|
||||
assert deepgram_listen_requested_model(query_string) == deepgram_listen_model(target)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_url", "expected"),
|
||||
[
|
||||
pytest.param(NOVA_3_URL, ("streaming/nova-3", "nova-3"), id="monolingual"),
|
||||
pytest.param(f"{NOVA_3_URL}&language=en", ("streaming/nova-3", "nova-3"), id="explicit language"),
|
||||
pytest.param(
|
||||
f"{NOVA_3_URL}&language=multi",
|
||||
("streaming/nova-3-multilingual", "streaming/nova-3", "nova-3"),
|
||||
id="multilingual",
|
||||
),
|
||||
pytest.param(
|
||||
f"{NOVA_3_URL}&language=MULTI",
|
||||
("streaming/nova-3-multilingual", "streaming/nova-3", "nova-3"),
|
||||
id="multilingual any case",
|
||||
),
|
||||
pytest.param(
|
||||
"wss://api.deepgram.com/v1/listen?model=nova-2&language=multi",
|
||||
("streaming/nova-2-multilingual", "streaming/nova-2", "nova-2"),
|
||||
id="other model",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_deepgram_listen_base_pricing_models(upstream_url: str, expected: tuple[str, ...]):
|
||||
assert deepgram_listen_base_pricing_models(upstream_url) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_url", "expected"),
|
||||
[
|
||||
pytest.param(NOVA_3_URL, (), id="no add-ons"),
|
||||
pytest.param(f"{NOVA_3_URL}&redact=pci", ("streaming/redact",), id="redact"),
|
||||
pytest.param(f"{NOVA_3_URL}&redact=pci&redact=ssn", ("streaming/redact",), id="repeated redact once"),
|
||||
pytest.param(f"{NOVA_3_URL}&keyterm=a&keyterm=b", ("streaming/keyterm",), id="keyterm"),
|
||||
pytest.param(f"{NOVA_3_URL}&detect_entities=true", ("streaming/detect_entities",), id="detect_entities"),
|
||||
pytest.param(f"{NOVA_3_URL}&diarize=true", ("streaming/diarize",), id="diarize"),
|
||||
pytest.param(f"{NOVA_3_URL}&diarize_model=v1", ("streaming/diarize",), id="diarize_model"),
|
||||
pytest.param(f"{NOVA_3_URL}&diarize=true&diarize_model=latest", ("streaming/diarize",), id="diarize both once"),
|
||||
pytest.param(f"{NOVA_3_URL}&detect_entities=false&diarize=FALSE&redact=", (), id="disabled"),
|
||||
pytest.param(
|
||||
f"{NOVA_3_URL}&detect_entities=false&detect_entities=true",
|
||||
("streaming/detect_entities",),
|
||||
id="any enabling value wins",
|
||||
),
|
||||
pytest.param(
|
||||
f"{NOVA_3_URL}&diarize=true&redact=pci&keyterm=x&detect_entities=true",
|
||||
("streaming/detect_entities", "streaming/diarize", "streaming/keyterm", "streaming/redact"),
|
||||
id="all, sorted",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_deepgram_listen_addon_pricing_models(upstream_url: str, expected: tuple[str, ...]):
|
||||
assert deepgram_listen_addon_pricing_models(upstream_url) == expected
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ from litellm.types.utils import StandardLoggingPayload, TranscriptionResponse
|
|||
|
||||
NOVA_3_URL: Final = "wss://api.deepgram.com/v1/listen?model=nova-3&encoding=linear16&sample_rate=16000"
|
||||
|
||||
pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map")
|
||||
|
||||
|
||||
def _results(start: object, duration: object, transcript: str = "", is_final: object = True) -> dict[str, object]:
|
||||
return {
|
||||
|
|
@ -63,13 +65,22 @@ def _logging_obj(call_id: str = "call-dg") -> LiteLLMLoggingObj:
|
|||
)
|
||||
|
||||
|
||||
def _registry_cost(model: str, seconds: float) -> float:
|
||||
def _registry_cost(pricing_model: str, seconds: float) -> float:
|
||||
"""Derives the expected charge from the live cost map rather than pinning a vendor price."""
|
||||
per_second: Final = litellm.model_cost[f"deepgram/{model}"]["input_cost_per_second"]
|
||||
per_second: Final = litellm.model_cost[f"deepgram/{pricing_model}"]["input_cost_per_second"]
|
||||
assert per_second > 0
|
||||
return per_second * seconds
|
||||
|
||||
|
||||
def _cost(upstream_url: str, *frames: dict[str, object]) -> float:
|
||||
handler_result = DeepgramListenPassthroughLoggingHandler().deepgram_listen_passthrough_handler(
|
||||
websocket_messages=frames, logging_obj=_logging_obj(), upstream_url=upstream_url
|
||||
)
|
||||
response_cost = handler_result["kwargs"]["response_cost"]
|
||||
assert isinstance(response_cost, float)
|
||||
return response_cost
|
||||
|
||||
|
||||
def test_handler_bills_metadata_duration_at_the_registry_rate_and_names_the_model():
|
||||
frames = (_results(0.0, 5.0, "first sentence"), _results(5.0, 7.5, "second sentence"), _metadata(12.5))
|
||||
logging_obj = _logging_obj()
|
||||
|
|
@ -85,15 +96,78 @@ def test_handler_bills_metadata_duration_at_the_registry_rate_and_names_the_mode
|
|||
assert isinstance(result, TranscriptionResponse)
|
||||
assert result.text == "first sentence second sentence"
|
||||
assert result._hidden_params["audio_transcription_duration"] == 12.5
|
||||
assert result._hidden_params["response_cost"] == pytest.approx(_registry_cost("nova-3", 12.5))
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("nova-3", 12.5))
|
||||
assert result._hidden_params["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 12.5))
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 12.5))
|
||||
assert handler_result["kwargs"]["model"] == "nova-3"
|
||||
assert handler_result["kwargs"]["custom_llm_provider"] == "deepgram"
|
||||
assert handler_result["kwargs"]["litellm_params"] == {"metadata": {}}
|
||||
assert logging_obj.model == "nova-3"
|
||||
assert logging_obj.model_call_details["model"] == "nova-3"
|
||||
assert logging_obj.model_call_details["custom_llm_provider"] == "deepgram"
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(_registry_cost("nova-3", 12.5))
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 12.5))
|
||||
|
||||
|
||||
def test_handler_bills_streaming_not_prerecorded_rates():
|
||||
"""Deepgram prices /v1/listen over a WebSocket separately from pre-recorded transcription, so the streaming entry
|
||||
must be the one charged; the two registry rows only need to differ for this to matter, whatever their values."""
|
||||
streaming = litellm.model_cost["deepgram/streaming/nova-3"]["input_cost_per_second"]
|
||||
prerecorded = litellm.model_cost["deepgram/nova-3"]["input_cost_per_second"]
|
||||
assert streaming != prerecorded
|
||||
|
||||
assert _cost(NOVA_3_URL, _metadata(60.0)) == pytest.approx(60.0 * streaming)
|
||||
|
||||
|
||||
def test_handler_bills_multilingual_streaming_when_language_is_multi():
|
||||
monolingual = _cost(NOVA_3_URL, _metadata(60.0))
|
||||
multilingual = _cost(f"{NOVA_3_URL}&language=multi", _metadata(60.0))
|
||||
|
||||
assert multilingual == pytest.approx(_registry_cost("streaming/nova-3-multilingual", 60.0))
|
||||
assert multilingual > monolingual
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "addons"),
|
||||
[
|
||||
pytest.param("redact=pci", ("redact",), id="redaction"),
|
||||
pytest.param("redact=pci&redact=numbers", ("redact",), id="redaction counted once"),
|
||||
pytest.param("keyterm=LiteLLM&keyterm=Deepgram", ("keyterm",), id="keyterm prompting"),
|
||||
pytest.param("detect_entities=true", ("detect_entities",), id="entity detection"),
|
||||
pytest.param("diarize=true", ("diarize",), id="diarization"),
|
||||
pytest.param("diarize_model=v1", ("diarize",), id="diarization via diarize_model"),
|
||||
pytest.param("diarize=true&diarize_model=v1", ("diarize",), id="diarization counted once"),
|
||||
pytest.param(
|
||||
"redact=pci&keyterm=x&detect_entities=true&diarize=true",
|
||||
("redact", "keyterm", "detect_entities", "diarize"),
|
||||
id="every add-on",
|
||||
),
|
||||
pytest.param("detect_entities=false&diarize=False&redact=", (), id="disabled add-ons cost nothing"),
|
||||
],
|
||||
)
|
||||
def test_handler_adds_each_priced_add_on_once_on_top_of_the_base_rate(query: str, addons: tuple[str, ...]):
|
||||
base = _cost(NOVA_3_URL, _metadata(60.0))
|
||||
expected = base + sum(_registry_cost(f"streaming/{addon}", 60.0) for addon in addons)
|
||||
|
||||
assert _cost(f"{NOVA_3_URL}&{query}", _metadata(60.0)) == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_handler_add_ons_scale_with_channels_like_the_base_rate():
|
||||
stereo_plain = _cost(f"{NOVA_3_URL}&channels=2", _metadata(60.0, channels=2))
|
||||
stereo_redacted = _cost(f"{NOVA_3_URL}&channels=2&redact=pci", _metadata(60.0, channels=2))
|
||||
|
||||
assert stereo_redacted - stereo_plain == pytest.approx(_registry_cost("streaming/redact", 120.0))
|
||||
|
||||
|
||||
def test_handler_falls_back_to_the_prerecorded_rate_for_a_model_without_a_streaming_entry():
|
||||
assert "deepgram/streaming/nova-2" not in litellm.model_cost
|
||||
|
||||
handler_result = DeepgramListenPassthroughLoggingHandler().deepgram_listen_passthrough_handler(
|
||||
websocket_messages=(_metadata(60.0),),
|
||||
logging_obj=_logging_obj(),
|
||||
upstream_url="wss://api.deepgram.com/v1/listen?model=nova-2",
|
||||
)
|
||||
|
||||
assert handler_result["kwargs"]["model"] == "nova-2"
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("nova-2", 60.0))
|
||||
|
||||
|
||||
def test_handler_falls_back_to_results_frames_when_the_stream_ends_without_metadata():
|
||||
|
|
@ -104,7 +178,7 @@ def test_handler_falls_back_to_results_frames_when_the_stream_ends_without_metad
|
|||
)
|
||||
|
||||
assert handler_result["result"]._hidden_params["audio_transcription_duration"] == 72.5
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("nova-3", 72.5))
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 72.5))
|
||||
|
||||
|
||||
def test_handler_charges_more_for_more_audio_on_the_same_model():
|
||||
|
|
@ -133,7 +207,7 @@ def test_handler_bills_every_channel_of_a_multichannel_session():
|
|||
|
||||
assert stereo["result"]._hidden_params["audio_transcription_duration"] == 60.0
|
||||
assert stereo["kwargs"]["response_cost"] == pytest.approx(2 * mono["kwargs"]["response_cost"])
|
||||
assert stereo["kwargs"]["response_cost"] == pytest.approx(_registry_cost("nova-3", 60.0))
|
||||
assert stereo["kwargs"]["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 60.0))
|
||||
|
||||
|
||||
def test_handler_bills_the_declared_channels_when_the_stream_dies_before_any_frame_reports_them():
|
||||
|
|
@ -144,7 +218,7 @@ def test_handler_bills_the_declared_channels_when_the_stream_dies_before_any_fra
|
|||
)
|
||||
|
||||
assert handler_result["result"]._hidden_params["audio_transcription_duration"] == 30.0
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("nova-3", 30.0))
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 30.0))
|
||||
|
||||
|
||||
def test_handler_keeps_the_spend_row_but_no_cost_for_an_unpriced_model():
|
||||
|
|
@ -226,6 +300,6 @@ async def test_success_handler_dispatches_deepgram_listen_and_logs_duration_base
|
|||
payload = capturing_logger.payloads[0]
|
||||
assert payload["model"] == "nova-3"
|
||||
assert payload["custom_llm_provider"] == "deepgram"
|
||||
assert payload["response_cost"] == pytest.approx(_registry_cost("nova-3", 20.0))
|
||||
assert payload["response_cost"] == pytest.approx(_registry_cost("streaming/nova-3", 20.0))
|
||||
assert payload["metadata"]["user_api_key_team_id"] == "team-stt"
|
||||
assert payload["id"] == "call-dg-e2e"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
"""Deepgram ``/v1/listen`` passthrough WebSocket route: registration, auth, credential injection, target URL."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
|
@ -12,13 +13,16 @@ from fastapi.testclient import TestClient
|
|||
from starlette.routing import WebSocketRoute
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy._lazy_features import LAZY_FEATURES
|
||||
from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_checks import _cache_key_object
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
_websocket_relay,
|
||||
deepgram_listen_websocket_route,
|
||||
router,
|
||||
)
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
GET_CREDENTIALS: Final = (
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials"
|
||||
|
|
@ -302,6 +306,62 @@ def test_deepgram_listen_authenticates_the_litellm_key_and_relays_to_deepgram(mo
|
|||
]
|
||||
|
||||
|
||||
async def _cache_restricted_key(virtual_key: str, models: list[str]) -> DualCache:
|
||||
cache = DualCache()
|
||||
await _cache_key_object(
|
||||
hashed_token=hash_token(virtual_key),
|
||||
user_api_key_obj=UserAPIKeyAuth(token=hash_token(virtual_key), models=models),
|
||||
user_api_key_cache=cache,
|
||||
proxy_logging_obj=None,
|
||||
)
|
||||
return cache
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expect_relay"),
|
||||
[
|
||||
pytest.param("model=nova-2", True, id="allowed model named"),
|
||||
pytest.param("model=nova-3", False, id="denied model named"),
|
||||
pytest.param("", False, id="model omitted, default denied"),
|
||||
pytest.param("model=&language=en", False, id="model blank, default denied"),
|
||||
],
|
||||
)
|
||||
def test_deepgram_listen_authorizes_the_model_it_will_actually_send_upstream(query, expect_relay, monkeypatch):
|
||||
"""A key allowed only ``nova-2`` must not reach ``nova-3`` by leaving ``model`` out and letting the proxy fill
|
||||
in its default: the real key auth path must see the same model the upstream target will carry."""
|
||||
monkeypatch.delenv("DEEPGRAM_API_BASE", raising=False)
|
||||
cache = asyncio.run(_cache_restricted_key("sk-only-nova-2", ["nova-2"]))
|
||||
relay = _FakeRelay()
|
||||
client = TestClient(_app_with_relay(relay))
|
||||
|
||||
with (
|
||||
patch(GET_CREDENTIALS, return_value="dg-provider-key"),
|
||||
patch.multiple( # test-quality-ok: the real key auth path reads these proxy_server globals and has no injection seam
|
||||
"litellm.proxy.proxy_server",
|
||||
master_key="sk-master",
|
||||
prisma_client=MagicMock(),
|
||||
user_api_key_cache=cache,
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
),
|
||||
):
|
||||
if expect_relay:
|
||||
with client.websocket_connect(
|
||||
f"/deepgram/v1/listen?{query}", headers={"Authorization": "Bearer sk-only-nova-2"}
|
||||
):
|
||||
pass
|
||||
assert [call.target for call in relay.calls] == [f"wss://api.deepgram.com/v1/listen?{query}"]
|
||||
return
|
||||
with pytest.raises(WebSocketDisconnect) as disconnect:
|
||||
with client.websocket_connect(
|
||||
f"/deepgram/v1/listen?{query}", headers={"Authorization": "Bearer sk-only-nova-2"}
|
||||
):
|
||||
pass
|
||||
|
||||
assert disconnect.value.code == 1008
|
||||
assert relay.calls == []
|
||||
|
||||
|
||||
def test_deepgram_listen_echoes_the_browser_subprotocol_that_carries_the_litellm_key(monkeypatch):
|
||||
"""Browsers cannot set headers, so they send the key as a subprotocol and abort the handshake unless the
|
||||
server echoes that subprotocol back; the key itself must still stay off the upstream connection."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue