From 2e8dc0a627b552d408910d84628692eca5452be3 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 03:03:19 +0000 Subject: [PATCH 1/7] feat(proxy): add Azure AI Speech pass-through route Adds /azure_speech/{endpoint:path}, an authenticated pass-through for the Azure AI Speech REST APIs: short-audio recognition on .stt.speech.microsoft.com and batch transcription on .api.cognitive.microsoft.com. The proxy resolves the subscription key through PassthroughEndpointRouter (AZURE_SPEECH_API_KEY or an Admin UI credential), picks the host from AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE, injects Ocp-Apim-Subscription-Key, strips the caller's Authorization and subscription-key headers, forwards the raw audio body byte for byte, and records a zero-cost SpendLogs row tagged azure_speech since the price map has no Azure Speech STT entry Resolves LIT-7939 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- gateway/routes/allowlist.py | 1 + helm/litellm/templates/ingress.yaml | 2 +- litellm/constants.py | 10 + litellm/passthrough/utils.py | 1 + litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 222 +++++++++++++ litellm/proxy/_types.py | 1 + litellm/proxy/auth/user_api_key_auth.py | 7 + .../proxy/common_utils/http_parsing_utils.py | 13 +- .../llm_passthrough_endpoints.py | 118 +++++++ ...zure_speech_passthrough_logging_handler.py | 84 +++++ .../pass_through_endpoints.py | 3 +- .../pass_through_endpoints/success_handler.py | 24 +- .../provider_create_fields.json | 18 ++ litellm/types/utils.py | 1 + terraform/litellm/aws/locals.tf | 2 +- terraform/litellm/gcp/locals.tf | 2 +- ...est_billable_request_metrics_middleware.py | 5 + ...zure_speech_passthrough_logging_handler.py | 123 ++++++++ .../test_llm_pass_through_endpoints.py | 294 ++++++++++++++++++ .../test_passthrough_endpoint_router.py | 16 + .../src/components/provider_info_helpers.tsx | 4 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 231 ++++++++++++++ 23 files changed, 1177 insertions(+), 6 deletions(-) create mode 100644 litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py create mode 100644 tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 099c6d5179f..5b8c44809fe 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -82,6 +82,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/anthropic/", "/azure/", "/azure_ai/", + "/azure_speech/", "/aws/", "/bedrock/", "/comprehendmedical", diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index d42558b9396..94ac4d2d8b0 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -66,7 +66,7 @@ "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search" "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat" "/v1beta" "/interactions" - "/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/comprehendmedical" "/cohere" "/gemini" "/google" + "/anthropic" "/azure" "/azure_ai" "/azure_speech" "/aws" "/bedrock" "/comprehendmedical" "/cohere" "/gemini" "/google" "/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm" "/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough" "/toolset" diff --git a/litellm/constants.py b/litellm/constants.py index 8409a161800..69de889a326 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1570,6 +1570,16 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = { # Works for all LLM pass-through endpoints (Vertex AI, Anthropic, Bedrock, etc.) PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-" +AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech" +AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech" +AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: Final = "/speech/" +AZURE_SPEECH_BATCH_PATH_PREFIX: Final = "/speechtotext/" +AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com" +AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com" +AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key" +AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio" +AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription" + BASE_MCP_ROUTE: Final = "/mcp" BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour diff --git a/litellm/passthrough/utils.py b/litellm/passthrough/utils.py index 7eb14fcc118..452c9c7de9d 100644 --- a/litellm/passthrough/utils.py +++ b/litellm/passthrough/utils.py @@ -17,6 +17,7 @@ _PASS_THROUGH_PROTECTED_HEADERS: Final[frozenset] = frozenset( "api-key", "x-api-key", "x-goog-api-key", + "ocp-apim-subscription-key", "host", "content-length", "accept-encoding", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index faf95397fa5..92cc014967d 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -196,6 +196,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/assemblyai/", "/azure/", "/azure_ai/", + "/azure_speech/", "/bedrock/", "/cohere/", "/comprehendmedical", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 74f38b3ca6d..dbfbc317d24 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -17133,6 +17133,228 @@ ] } }, + "/azure_speech/{endpoint}": { + "delete": { + "description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)", + "operationId": "azure_speech_proxy_route_azure_speech__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Azure Speech Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)", + "operationId": "azure_speech_proxy_route_azure_speech__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Azure Speech Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)", + "operationId": "azure_speech_proxy_route_azure_speech__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Azure Speech Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)", + "operationId": "azure_speech_proxy_route_azure_speech__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Azure Speech Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)", + "operationId": "azure_speech_proxy_route_azure_speech__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Azure Speech Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/bedrock/{endpoint}": { "delete": { "description": "This is the v1 passthrough for Bedrock.\nV2 is handled by the `/bedrock/v2` endpoint.\n[Docs](https://docs.litellm.ai/docs/pass_through/bedrock)", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 680c63393e8..489186a8f69 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -468,6 +468,7 @@ class LiteLLMRoutes(enum.Enum): mapped_pass_through_routes = [ "/bedrock", "/comprehendmedical", + "/azure_speech", "/vertex-ai", "/vertex_ai", "/cohere", diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 4cbd4213463..5f2e0a3c1a8 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -105,6 +105,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, _safe_get_request_query_params, _safe_set_request_parsed_body, + is_opaque_audio_pass_through_request, populate_request_with_path_params, read_raw_json_body, rewrite_request_model, @@ -1354,6 +1355,12 @@ async def _read_request_body_deferring_parse_failure( must run (resolving identity onto the request's trace) before the 400 goes out; the caller re-raises the returned exception once identity is seeded. """ + if is_opaque_audio_pass_through_request( + route=get_request_route(request=request), + content_type=_safe_get_request_headers(request=request).get("content-type", ""), + ): + _safe_set_request_parsed_body(request=request, parsed_body={}) # mutable-ok: the body cache stores a plain dict + return {}, None # mutable-ok: request_data is a plain dict across the whole auth path try: parsed_body: Final = await _read_request_body(request=request) except ProxyException as parse_exception: diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index f5b6a0a766d..29dc36f3dba 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -9,7 +9,12 @@ from fastapi import Request, UploadFile, status from typing_extensions import NotRequired, ReadOnly, Required from litellm._logging import verbose_proxy_logger -from litellm.constants import CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB +from litellm.constants import ( + AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX, + AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, + CLIENT_REQUESTED_MODEL_SCOPE_KEY, + MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB, +) from litellm.proxy._types import ProxyException from litellm.proxy.common_utils.callback_utils import ( get_metadata_variable_name_from_kwargs, @@ -214,6 +219,12 @@ async def _read_request_body(request: Request | None) -> dict: return {} +def is_opaque_audio_pass_through_request(route: str, content_type: str) -> bool: + return route.startswith( + f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}{AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX}" + ) and _normalize_media_type(content_type).startswith("audio/") + + async def read_raw_json_body(request: Request | None) -> bytes | None: if request is None or _safe_get_request_parsed_body(request=request) is None: return None diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index b9b8cb3a22b..e64eac87a7f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -30,6 +30,13 @@ from litellm import get_llm_provider from litellm._logging import verbose_proxy_logger from litellm.constants import ( ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS, + AZURE_SPEECH_BATCH_PATH_PREFIX, + AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN, + AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX, + AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, + AZURE_SPEECH_STT_DOMAIN, + AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix @@ -1316,6 +1323,117 @@ async def comprehend_medical_sdk_proxy_route( ) +AZURE_SPEECH_FORWARDED_REQUEST_HEADERS: Final = ("content-type", "accept") +AZURE_SPEECH_ENDPOINT_FAMILY_DOMAINS: Final = MappingProxyType( + { + AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: AZURE_SPEECH_STT_DOMAIN, + AZURE_SPEECH_BATCH_PATH_PREFIX: AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN, + } +) + + +def resolve_azure_speech_base_url(endpoint_path: str, api_base: str | None, region: str | None) -> httpx.URL | None: + """ + Azure AI Speech serves the two REST families from different regional hosts: short-audio + recognition under ``{region}.stt.speech.microsoft.com`` and batch transcription under + ``{region}.api.cognitive.microsoft.com``. An operator-configured ``api_base`` (custom + domain or private endpoint) serves both and wins over the region. Returns ``None`` when + the path is outside both families so the operator key is never sent for an unknown API. + """ + domain: Final = next( + ( + family_domain + for family_prefix, family_domain in AZURE_SPEECH_ENDPOINT_FAMILY_DOMAINS.items() + if endpoint_path.startswith(family_prefix) + ), + None, + ) + if domain is None: + return None + if api_base: + return httpx.URL(api_base) + if not region: + return None + return httpx.URL(f"https://{region}.{domain}") + + +@router.api_route( + f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list + tags=["Azure AI Speech Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list +) +async def azure_speech_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + """ + Pass-through for the Azure AI Speech REST APIs (speech to text), e.g. + `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US` + with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`. + + The body is forwarded byte for byte and the proxy injects its own + `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key + and is never forwarded. + + [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) + """ + endpoint_path: Final = httpx.URL(endpoint).path + normalized_endpoint_path: Final = endpoint_path if endpoint_path.startswith("/") else f"/{endpoint_path}" + base_url: Final = resolve_azure_speech_base_url( + endpoint_path=normalized_endpoint_path, + api_base=get_secret_str(secret_name="AZURE_SPEECH_API_BASE"), + region=get_secret_str(secret_name="AZURE_SPEECH_REGION"), + ) + if base_url is None: + raise HTTPException( + status_code=400, + detail=( + f"Unsupported Azure Speech path: {normalized_endpoint_path}. Supported prefixes are " + f"{AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX} and {AZURE_SPEECH_BATCH_PATH_PREFIX}; set " + "AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE in the proxy environment." + ), + ) + azure_speech_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + region_name=None, + ) + if azure_speech_api_key is None: + raise HTTPException( + status_code=400, + detail="Azure Speech credentials not found. Set AZURE_SPEECH_API_KEY in the proxy environment.", + ) + + target_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint_path) + ) + request_headers: Final = _safe_get_request_headers(request) + upstream_headers: Final = MappingProxyType( + { + header_name: header_value + for header_name, header_value in ( + *( + (header_name, request_headers[header_name]) + for header_name in AZURE_SPEECH_FORWARDED_REQUEST_HEADERS + if header_name in request_headers + ), + (AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, azure_speech_api_key), + ) + } + ) + raw_body: Final = await request.body() + + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(target_url), + custom_headers=upstream_headers, + custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + ) + setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, raw_body) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + def _resolve_vertex_model_from_router( model_id: str, llm_router: litellm.Router | None, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py new file mode 100644 index 00000000000..a7084a9545e --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -0,0 +1,84 @@ +from collections.abc import Mapping +from datetime import datetime +from typing import Final +from urllib.parse import urlparse + +import httpx + +from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + AZURE_SPEECH_BATCH_MODEL, + AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + AZURE_SPEECH_SHORT_AUDIO_MODEL, + AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, +) +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.utils import StandardPassThroughResponseObject + + +class AzureSpeechPassthroughLoggingHandler: + @staticmethod + def _model_from_url_route(url_route: str) -> str: + path: Final = urlparse(url_route).path + if path.startswith(AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX): + return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_SHORT_AUDIO_MODEL}" + return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}" + + @staticmethod + def azure_speech_passthrough_handler( + httpx_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + url_route: str, + result: str, + start_time: datetime, + end_time: datetime, + cache_hit: bool, + request_body: Mapping[str, object], + **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler + ) -> PassThroughEndpointLoggingTypedDict: + """ + Records model and provider for an Azure AI Speech REST call. Azure bills per audio + hour after the fact and neither the short-audio response nor the batch job carries + a billable duration this path can trust, so response_cost is recorded as 0.0 rather + than estimated. + """ + try: + model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) + + updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict + **kwargs, + "model": model_name, + "custom_llm_provider": AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + "response_cost": 0.0, + } + logging_obj.model_call_details.update( + model=model_name, + custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + response_cost=0.0, + ) + + standard_logging_object: Final = get_standard_logging_object_payload( + kwargs=updated_kwargs, + init_response_obj=StandardPassThroughResponseObject(response=result), + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + status="success", + ) + + handler_payload: Final[PassThroughEndpointLoggingTypedDict] = { + "result": StandardPassThroughResponseObject(response=result), + "kwargs": {**updated_kwargs, "standard_logging_object": standard_logging_object}, + } + except Exception as e: # noqa: BLE001 # logging must never fail the forwarded request + verbose_proxy_logger.exception("Error in Azure Speech passthrough logging handler: %s", e) + fallback_payload: Final[PassThroughEndpointLoggingTypedDict] = { + "result": StandardPassThroughResponseObject(response=result), + "kwargs": kwargs, + } + return fallback_payload + return handler_payload diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 685c19062bb..81268a7cf6e 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -52,6 +52,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_no_callback_env_reference from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import _get_masked_values from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.base_llm.managed_resources.utils import ( @@ -1023,7 +1024,7 @@ async def pass_through_request( verbose_proxy_logger.debug( "Pass through endpoint sending request to \nURL %s\nheaders: %s\nbody: %s\n", url, - upstream_headers, + _get_masked_values(upstream_headers), _parsed_body, ) diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 76a471302f4..919de5c1088 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -5,6 +5,7 @@ from urllib.parse import urlparse import httpx +from litellm.constants import AZURE_SPEECH_CUSTOM_LLM_PROVIDER from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import PassThroughEndpointLoggingResultValues from litellm.types.passthrough_endpoints.pass_through_endpoints import ( @@ -256,6 +257,24 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = comprehend_medical_handler_result["result"] # rebind-ok: elif-chain kwargs = comprehend_medical_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif self.is_azure_speech_route(custom_llm_provider): + from .llm_provider_handlers.azure_speech_passthrough_logging_handler import ( + AzureSpeechPassthroughLoggingHandler, + ) + + azure_speech_handler_result: Final = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=httpx_response, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) + standard_logging_response_object = azure_speech_handler_result["result"] # rebind-ok: elif-chain + kwargs = azure_speech_handler_result["kwargs"] # rebind-ok: elif-chain contract elif self.is_vertex_ai_live_route(url_route): from .llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import ( VertexAILivePassthroughLoggingHandler, @@ -300,7 +319,7 @@ class PassThroughEndpointLogging: ): standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None logging_obj.model_call_details["passthrough_logging_payload"] = passthrough_logging_payload - if self.is_assemblyai_route(url_route): + if self.is_assemblyai_route(url_route) and not self.is_azure_speech_route(custom_llm_provider): if AssemblyAIPassthroughLoggingHandler._should_log_request(httpx_response.request.method) is not True: return self.assemblyai_passthrough_logging_handler.assemblyai_passthrough_logging_handler( @@ -389,6 +408,9 @@ class PassThroughEndpointLogging: def is_comprehend_medical_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == "comprehendmedical" + def is_azure_speech_route(self, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == AZURE_SPEECH_CUSTOM_LLM_PROVIDER + def is_langfuse_route(self, url_route: str): parsed_url: Final = urlparse(url_route) for route in self.TRACKED_LANGFUSE_ROUTES: diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index cd781abee26..e6673ec99aa 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -586,6 +586,24 @@ ], "default_model_placeholder": "azure_ai/command-r-plus" }, + { + "provider": "Azure_Speech", + "provider_display_name": "Azure AI Speech", + "litellm_provider": "azure_speech", + "credential_fields": [ + { + "key": "api_key", + "label": "Azure AI Speech Subscription Key", + "placeholder": null, + "tooltip": "The Ocp-Apim-Subscription-Key for your Azure AI Speech resource. The proxy injects it on every /azure_speech/* pass-through request. Region and API base come from AZURE_SPEECH_REGION / AZURE_SPEECH_API_BASE", + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "azure_speech/short-audio" + }, { "provider": "AZURE_TEXT", "provider_display_name": "Azure Text", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index aaa16fd2d44..3c2c549e89e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4060,6 +4060,7 @@ class LlmProviders(str, Enum): TOPAZ = "topaz" SAP_GENERATIVE_AI_HUB = "sap" ASSEMBLYAI = "assemblyai" + AZURE_SPEECH = "azure_speech" CHARITY_ENGINE = "charity_engine" GITHUB_COPILOT = "github_copilot" SNOWFLAKE = "snowflake" diff --git a/terraform/litellm/aws/locals.tf b/terraform/litellm/aws/locals.tf index bd5b97b0f50..fcf1f7b905f 100644 --- a/terraform/litellm/aws/locals.tf +++ b/terraform/litellm/aws/locals.tf @@ -86,7 +86,7 @@ locals { "/queue/chat/*", "/v1beta/*", "/interactions/*", - "/anthropic/*", "/azure/*", "/azure_ai/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", + "/anthropic/*", "/azure/*", "/azure_ai/*", "/azure_speech/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", "/cohere/*", "/gemini/*", "/google/*", "/vertex_ai/*", "/vertex-ai/*", "/assemblyai/*", "/eu.assemblyai/*", diff --git a/terraform/litellm/gcp/locals.tf b/terraform/litellm/gcp/locals.tf index 3861413d496..dca7b05f1c9 100644 --- a/terraform/litellm/gcp/locals.tf +++ b/terraform/litellm/gcp/locals.tf @@ -55,7 +55,7 @@ locals { "/queue/chat/*", "/v1beta/*", "/interactions/*", - "/anthropic/*", "/azure/*", "/azure_ai/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", + "/anthropic/*", "/azure/*", "/azure_ai/*", "/azure_speech/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", "/cohere/*", "/gemini/*", "/google/*", "/vertex_ai/*", "/vertex-ai/*", "/assemblyai/*", "/eu.assemblyai/*", diff --git a/tests/test_litellm/proxy/middleware/test_billable_request_metrics_middleware.py b/tests/test_litellm/proxy/middleware/test_billable_request_metrics_middleware.py index 9c61412bd6e..5ce7aa858c1 100644 --- a/tests/test_litellm/proxy/middleware/test_billable_request_metrics_middleware.py +++ b/tests/test_litellm/proxy/middleware/test_billable_request_metrics_middleware.py @@ -116,6 +116,11 @@ def test_is_pure_asgi_not_base_http_middleware(): # Bare AWS-SDK-shaped route carries the operation in X-Amz-Target and writes SpendLogs ("/comprehendmedical", (BillableCategory.LLM, "/comprehendmedical")), ("/comprehendmedical/DetectEntitiesV2", (BillableCategory.LLM, "/comprehendmedical")), + ( + "/azure_speech/speech/recognition/conversation/cognitiveservices/v1", + (BillableCategory.LLM, "/azure_speech"), + ), + ("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")), ("/mcp", (BillableCategory.MCP, "/mcp")), ("/mcp/", (BillableCategory.MCP, "/mcp")), ("/mcp/tools/list", (BillableCategory.MCP, "/mcp")), diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py new file mode 100644 index 00000000000..91f7bf94281 --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -0,0 +1,123 @@ +from datetime import datetime +from unittest.mock import MagicMock + +import httpx +import pytest + +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.azure_speech_passthrough_logging_handler import ( + AzureSpeechPassthroughLoggingHandler, +) +from litellm.proxy.pass_through_endpoints.success_handler import ( + PassThroughEndpointLogging, +) + +SHORT_AUDIO_URL = "https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US" +BATCH_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2/transcriptions" +TRANSCRIPT = '{"RecognitionStatus":"Success","DisplayText":"Hello world."}' + + +def _make_response(url: str) -> httpx.Response: + request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}) + return httpx.Response(200, request=request, text=TRANSCRIPT) + + +def _make_logging_obj() -> MagicMock: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "test-call-id" + logging_obj.model_call_details = {} + return logging_obj + + +class TestAzureSpeechPassthroughHandler: + @pytest.mark.parametrize( + "url_route,expected_model", + [ + (SHORT_AUDIO_URL, "azure_speech/short-audio"), + (BATCH_URL, "azure_speech/batch-transcription"), + (f"{BATCH_URL}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files", "azure_speech/batch-transcription"), + ], + ) + def test_records_model_provider_and_zero_cost(self, url_route: str, expected_model: str): + logging_obj = _make_logging_obj() + + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(url_route), + logging_obj=logging_obj, + url_route=url_route, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["result"] == {"response": TRANSCRIPT} + assert handler_result["kwargs"]["model"] == expected_model + assert handler_result["kwargs"]["custom_llm_provider"] == "azure_speech" + assert handler_result["kwargs"]["response_cost"] == 0.0 + assert handler_result["kwargs"]["standard_logging_object"]["response_cost"] == 0.0 + assert handler_result["kwargs"]["standard_logging_object"]["model"] == expected_model + assert logging_obj.model_call_details["model"] == expected_model + assert logging_obj.model_call_details["custom_llm_provider"] == "azure_speech" + assert logging_obj.model_call_details["response_cost"] == 0.0 + + def test_subscription_key_never_reaches_the_logging_payload(self): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL), + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert "server-secret" not in repr(handler_result) + + +class TestIsAzureSpeechRoute: + def test_matches_by_provider_tag(self): + assert PassThroughEndpointLogging().is_azure_speech_route("azure_speech") + + @pytest.mark.parametrize("provider", ["azure", "azure_ai", "comprehendmedical", None]) + def test_does_not_match_other_providers(self, provider: str | None): + assert not PassThroughEndpointLogging().is_azure_speech_route(provider) + + def test_config_driven_passthrough_to_azure_speech_host_is_not_claimed(self): + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_make_response(SHORT_AUDIO_URL), + response_body={"RecognitionStatus": "Success"}, + request_body={}, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider=None, + ) + + assert normalized["kwargs"].get("model") != "azure_speech/short-audio" + assert "response_cost" not in normalized["kwargs"] + + +class TestNormalizeDispatch: + def test_normalize_routes_to_azure_speech_handler(self): + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_make_response(SHORT_AUDIO_URL), + response_body={"RecognitionStatus": "Success"}, + request_body={}, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="azure_speech", + ) + + assert normalized["standard_logging_response_object"] == {"response": TRANSCRIPT} + assert normalized["kwargs"]["model"] == "azure_speech/short-audio" + assert normalized["kwargs"]["custom_llm_provider"] == "azure_speech" + assert normalized["kwargs"]["response_cost"] == 0.0 diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 6e82c90514d..9fe7f5b6ee9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2,6 +2,7 @@ import asyncio import base64 import contextlib import json +import logging import os import traceback from collections.abc import Iterator, Mapping @@ -6136,3 +6137,296 @@ class TestAzureRelayDeploymentSegment: ) assert [call["model"] for call in captured] == ["gpt", "gpt"] + + +AZURE_SPEECH_SHORT_AUDIO_ENDPOINT: Final = "/speech/recognition/conversation/cognitiveservices/v1" +AZURE_SPEECH_BATCH_ENDPOINT: Final = "/speechtotext/v3.2/transcriptions" +AZURE_SPEECH_PCM16_HEADER: Final = ( + b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00" +) +AZURE_SPEECH_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + b"\x00" * 3072 +AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range(256)) * 12 +AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."} + + +@pytest.fixture +def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") + monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus") + monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + +class TestAzureSpeechProxyRoute: + """Drives the real FastAPI route with respx standing in for the Azure hosts only.""" + + def test_short_audio_forwards_raw_wav_bytes_with_server_key(self, azure_speech_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post( + f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}" + ).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT)) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + params={"language": "en-US", "format": "detailed"}, + content=AZURE_SPEECH_WAV_BYTES, + headers={ + "Content-Type": "audio/wav; codecs=audio/pcm; samplerate=16000", + "Authorization": "Bearer sk-virtual", + "Ocp-Apim-Subscription-Key": "caller-supplied-key", + "x-pass-ocp-apim-subscription-key": "caller-supplied-key", + }, + ) + + assert (response.status_code, response.json()) == (200, AZURE_SPEECH_TRANSCRIPT) + sent = route.calls.last.request + assert sent.content == AZURE_SPEECH_WAV_BYTES + assert dict(sent.url.params) == {"language": "en-US", "format": "detailed"} + assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" + assert sent.headers["content-type"] == "audio/wav; codecs=audio/pcm; samplerate=16000" + assert "authorization" not in sent.headers + assert "caller-supplied-key" not in repr(sent.headers) + + def test_batch_json_goes_to_the_cognitive_services_host(self, azure_speech_client: TestClient) -> None: + body: Final = {"contentUrls": ["https://example.com/a.wav"], "locale": "en-US", "displayName": "job"} + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( + return_value=httpx.Response(201, json={"self": "https://eastus.api.cognitive.microsoft.com/x"}) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", + json=body, + headers={"Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 201 + sent = route.calls.last.request + assert json.loads(sent.content) == body + assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" + assert "authorization" not in sent.headers + + def test_batch_multipart_upload_is_forwarded_byte_for_byte(self, azure_speech_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( + return_value=httpx.Response(201, json={"status": "NotStarted"}) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", + files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")}, + data={"definition": json.dumps({"locales": ["en-US"]})}, + headers={"Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 201 + sent = route.calls.last.request + assert sent.headers["content-type"].startswith("multipart/form-data; boundary=") + assert AZURE_SPEECH_NON_UTF8_WAV_BYTES in sent.content + assert b'name="definition"' in sent.content + assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" + assert "authorization" not in sent.headers + + def test_batch_get_is_forwarded_with_the_job_id_path(self, azure_speech_client: TestClient) -> None: + job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files" + with respx.mock(assert_all_called=True) as upstream: + route = upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock( + return_value=httpx.Response(200, json={"values": []}) + ) + + response = azure_speech_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}) + + assert (response.status_code, response.json()) == (200, {"values": []}) + assert route.calls.last.request.headers["ocp-apim-subscription-key"] == "server-subscription-key" + + @pytest.mark.parametrize("method", ["GET", "POST"]) + def test_batch_requests_are_logged_as_azure_speech_not_assemblyai( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch, method: str + ) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.payloads.append(kwargs["standard_logging_object"]) + + recorder: Final = _Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder]) + with respx.mock(assert_all_called=True) as upstream: + upstream.request(method, f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"values": []}) + ) + + response = azure_speech_client.request( + method, + f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", + json={"locale": "en-US"} if method == "POST" else None, + headers={"Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 200 + assert [(p["model"], p["custom_llm_provider"], p["response_cost"]) for p in recorder.payloads] == [ + ("azure_speech/batch-transcription", "azure_speech", 0.0) + ] + + def test_api_base_wins_over_region_for_both_families( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("AZURE_SPEECH_API_BASE", "https://my-speech.cognitiveservices.azure.com") + with respx.mock(assert_all_called=True) as upstream: + short_audio = upstream.post( + f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}" + ).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT)) + batch = upstream.get(f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"values": []}) + ) + + azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + content=AZURE_SPEECH_WAV_BYTES, + headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, + ) + azure_speech_client.get(f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", headers={"Authorization": "Bearer x"}) + + assert short_audio.called and batch.called + + @pytest.mark.parametrize("endpoint", ["openai/deployments/whisper/audio/transcriptions", "speech", "speechtotext"]) + def test_unknown_path_family_is_rejected_before_any_upstream_call( + self, azure_speech_client: TestClient, endpoint: str + ) -> None: + with respx.mock(assert_all_called=False) as upstream: + catch_all = upstream.route().mock(return_value=httpx.Response(200)) + + response = azure_speech_client.post( + f"/azure_speech/{endpoint}", content=b"x", headers={"Authorization": "Bearer sk-virtual"} + ) + + assert response.status_code == 400 + assert not catch_all.called + + def test_missing_region_and_base_is_rejected_before_any_upstream_call( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("AZURE_SPEECH_REGION") + with respx.mock(assert_all_called=False) as upstream: + catch_all = upstream.route().mock(return_value=httpx.Response(200)) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + content=AZURE_SPEECH_WAV_BYTES, + headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 400 + assert "AZURE_SPEECH_REGION" in response.text + assert not catch_all.called + + def test_missing_api_key_is_rejected_before_any_upstream_call( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("AZURE_SPEECH_API_KEY") + with respx.mock(assert_all_called=False) as upstream: + catch_all = upstream.route().mock(return_value=httpx.Response(200)) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + content=AZURE_SPEECH_WAV_BYTES, + headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 400 + assert "AZURE_SPEECH_API_KEY" in response.text + assert not catch_all.called + + def test_azure_speech_is_a_mapped_pass_through_route(self) -> None: + from litellm.proxy._types import LiteLLMRoutes + + assert "/azure_speech" in LiteLLMRoutes.mapped_pass_through_routes.value + + +def _azure_speech_real_auth_attrs() -> dict[str, object]: + from litellm.caching.caching import DualCache + from litellm.proxy.utils import ProxyLogging + + user_api_key_cache: Final = DualCache() + return { + "prisma_client": None, + "user_api_key_cache": user_api_key_cache, + "proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache), + "master_key": "sk-master-key", + "general_settings": {}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "user_custom_auth": None, + "jwt_handler": None, + } + + +class TestAzureSpeechRawBodyThroughRealAuth: + """user_api_key_auth reads the body before the route runs; raw audio must not be parsed as JSON.""" + + def _post_wav( + self, monkeypatch: pytest.MonkeyPatch, path: str, api_key: str, body: bytes = AZURE_SPEECH_WAV_BYTES + ) -> httpx.Response: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") + monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus") + monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with patch.multiple( # test-quality-ok: the real user_api_key_auth reads proxy_server module globals (master_key, caches) that have no injection seam + "litellm.proxy.proxy_server", **_azure_speech_real_auth_attrs() + ): + client = TestClient(app) + return client.post( + path, + params={"language": "en-US"}, + content=body, + headers={"Content-Type": "audio/wav", "Authorization": f"Bearer {api_key}"}, + ) + + @pytest.mark.parametrize("body", [AZURE_SPEECH_WAV_BYTES, AZURE_SPEECH_NON_UTF8_WAV_BYTES], ids=["ascii", "binary"]) + def test_master_key_with_raw_wav_body_reaches_azure_without_a_parse_attempt( + self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, body: bytes + ) -> None: + with respx.mock(assert_all_called=True) as upstream, caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + route = upstream.post( + f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}" + ).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT)) + + response = self._post_wav( + monkeypatch, f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", "sk-master-key", body=body + ) + + assert (response.status_code, response.json()) == (200, AZURE_SPEECH_TRANSCRIPT) + assert route.calls.last.request.content == body + assert [record.message for record in caplog.records if "request body" in record.message] == [] + + def test_wrong_litellm_key_with_raw_wav_body_is_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None: + with respx.mock(assert_all_called=False) as upstream: + catch_all = upstream.route().mock(return_value=httpx.Response(200)) + + response = self._post_wav(monkeypatch, f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", "sk-wrong") + + assert response.status_code in (400, 401), response.text + assert not catch_all.called + + @pytest.mark.parametrize("path", ["/v1/chat/completions", f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}"]) + def test_audio_content_type_off_the_short_audio_route_is_still_parsed_as_json( + self, monkeypatch: pytest.MonkeyPatch, path: str + ) -> None: + response = self._post_wav(monkeypatch, path, "sk-master-key", body=b'{}{"model": "gpt-4o"}') + + assert response.status_code == 400 + assert "Invalid JSON payload" in response.text diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index e3cbc2d507f..7a272a49853 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -159,6 +159,22 @@ def test_assemblyai_region_matching(): assert passthrough_router.get_credentials(custom_llm_provider="assemblyai", region_name=None) == "sk-us" +def test_azure_speech_dashboard_credential_resolves_through_flagged_deployment(monkeypatch): + monkeypatch.delenv("AZURE_SPEECH_API_KEY", raising=False) + CredentialAccessor.upsert_credentials([_credential("azure-speech-prod", "azure-subscription-key")]) + llm_router = litellm.Router( + model_list=[ + _flagged_deployment("azure_speech/short-audio", litellm_credential_name="azure-speech-prod"), + ] + ) + passthrough_router = _passthrough_router(llm_router) + + assert ( + passthrough_router.get_credentials(custom_llm_provider="azure_speech", region_name=None) + == "azure-subscription-key" + ) + + def test_env_fallback_when_no_router(monkeypatch): passthrough_router = _passthrough_router(None) monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env") diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index de83cd00790..1a1a2fd73aa 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -80,6 +80,7 @@ export enum Providers { SageMaker = "AWS SageMaker", Azure = "Azure", Azure_AI_Studio = "Azure AI Foundry (Studio)", + Azure_Speech = "Azure AI Speech", AZURE_TEXT = "Azure Text", BASETEN = "Baseten", BYTEZ = "Bytez", @@ -193,6 +194,7 @@ export const provider_map: Record = { AUTO_ROUTER: "auto_router", Azure: "azure", Azure_AI_Studio: "azure_ai", + Azure_Speech: "azure_speech", AZURE_TEXT: "azure_text", BASETEN: "baseten", Bedrock: "bedrock", @@ -310,6 +312,7 @@ export const providerLogoMap: Partial> = { [Providers.AssemblyAI]: assemblyaiSmallLogo.src, [Providers.Azure]: microsoftAzureLogo.src, [Providers.Azure_AI_Studio]: microsoftAzureLogo.src, + [Providers.Azure_Speech]: microsoftAzureLogo.src, [Providers.AZURE_TEXT]: microsoftAzureLogo.src, [Providers.BASETEN]: basetenLogo.src, [Providers.Bedrock]: bedrockLogo.src, @@ -427,6 +430,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Anthropic]: "claude-3-opus", [Providers.Azure]: "my-deployment", [Providers.Azure_AI_Studio]: "azure_ai/command-r-plus", + [Providers.Azure_Speech]: "azure_speech/short-audio", [Providers.Bedrock]: "claude-3-opus", [Providers.CHATGPT]: "chatgpt/gpt-5.4", [Providers.Cognition]: "cognition/swe-1.7", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 872875cc535..62b9921302a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1612,6 +1612,82 @@ export interface paths { patch: operations["azure_proxy_route_azure_ai__endpoint__patch"]; trace?: never; }; + "/azure_speech/{endpoint}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Azure Speech Proxy Route + * @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g. + * `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US` + * with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`. + * + * The body is forwarded byte for byte and the proxy injects its own + * `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key + * and is never forwarded. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) + */ + get: operations["azure_speech_proxy_route_azure_speech__endpoint__get"]; + /** + * Azure Speech Proxy Route + * @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g. + * `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US` + * with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`. + * + * The body is forwarded byte for byte and the proxy injects its own + * `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key + * and is never forwarded. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) + */ + put: operations["azure_speech_proxy_route_azure_speech__endpoint__put"]; + /** + * Azure Speech Proxy Route + * @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g. + * `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US` + * with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`. + * + * The body is forwarded byte for byte and the proxy injects its own + * `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key + * and is never forwarded. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) + */ + post: operations["azure_speech_proxy_route_azure_speech__endpoint__post"]; + /** + * Azure Speech Proxy Route + * @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g. + * `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US` + * with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`. + * + * The body is forwarded byte for byte and the proxy injects its own + * `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key + * and is never forwarded. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) + */ + delete: operations["azure_speech_proxy_route_azure_speech__endpoint__delete"]; + options?: never; + head?: never; + /** + * Azure Speech Proxy Route + * @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g. + * `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US` + * with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`. + * + * The body is forwarded byte for byte and the proxy injects its own + * `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key + * and is never forwarded. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) + */ + patch: operations["azure_speech_proxy_route_azure_speech__endpoint__patch"]; + trace?: never; + }; "/batches": { parameters: { query?: never; @@ -43394,6 +43470,161 @@ export interface operations { }; }; }; + azure_speech_proxy_route_azure_speech__endpoint__get: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + azure_speech_proxy_route_azure_speech__endpoint__put: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + azure_speech_proxy_route_azure_speech__endpoint__post: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + azure_speech_proxy_route_azure_speech__endpoint__delete: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + azure_speech_proxy_route_azure_speech__endpoint__patch: { + parameters: { + query?: never; + header?: never; + path: { + endpoint: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; list_batches_batches_get: { parameters: { query?: { From f2305879d06073dfa2fa9f8001659c48ae61e7d1 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 03:31:29 +0000 Subject: [PATCH 2/7] feat(proxy): price Azure Speech short audio pass-through from the recognized duration Short audio responses carry Offset and Duration in 100ns ticks; convert their sum to seconds and price it with the existing azure/speech/azure-stt entry through transcription_cost. Batch calls and responses without an integer duration stay at zero cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 + ...zure_speech_passthrough_logging_handler.py | 54 ++++++++--- .../pass_through_endpoints/success_handler.py | 1 + ...zure_speech_passthrough_logging_handler.py | 95 ++++++++++++++++--- .../test_llm_pass_through_endpoints.py | 43 +++++++++ 5 files changed, 171 insertions(+), 24 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 69de889a326..15a1d054e26 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1579,6 +1579,8 @@ AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com" AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key" AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio" AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription" +AZURE_SPEECH_PRICING_MODEL: Final = "azure/speech/azure-stt" +AZURE_SPEECH_TICKS_PER_SECOND: Final = 10_000_000 BASE_MCP_ROUTE: Final = "/mcp" diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py index a7084a9545e..74587acd453 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from typing import Final from urllib.parse import urlparse @@ -9,9 +9,12 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( AZURE_SPEECH_BATCH_MODEL, AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + AZURE_SPEECH_PRICING_MODEL, AZURE_SPEECH_SHORT_AUDIO_MODEL, AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, + AZURE_SPEECH_TICKS_PER_SECOND, ) +from litellm.cost_calculator import transcription_cost from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -21,16 +24,50 @@ from litellm.types.utils import StandardPassThroughResponseObject class AzureSpeechPassthroughLoggingHandler: + @staticmethod + def _is_short_audio_route(url_route: str) -> bool: + return urlparse(url_route).path.startswith(AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX) + @staticmethod def _model_from_url_route(url_route: str) -> str: - path: Final = urlparse(url_route).path - if path.startswith(AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX): + if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_SHORT_AUDIO_MODEL}" return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}" + @staticmethod + def _recognized_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float: + if not isinstance(response_body, Mapping): + return 0.0 + offset: Final = response_body.get("Offset") + duration: Final = response_body.get("Duration") + if not isinstance(offset, int) or not isinstance(duration, int): + return 0.0 + return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND + + @staticmethod + def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: + if not AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): + return 0.0 + audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body) + if audio_seconds <= 0.0: + return 0.0 + try: + prompt_cost, completion_cost = transcription_cost( + model=AZURE_SPEECH_PRICING_MODEL, + custom_llm_provider="azure", + duration=audio_seconds, + ) + except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row + verbose_proxy_logger.warning( + "No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e + ) + return 0.0 + return prompt_cost + completion_cost + @staticmethod def azure_speech_passthrough_handler( httpx_response: httpx.Response, + response_body: Mapping[str, object] | Sequence[object] | None, logging_obj: LiteLLMLoggingObj, url_route: str, result: str, @@ -40,25 +77,20 @@ class AzureSpeechPassthroughLoggingHandler: request_body: Mapping[str, object], **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler ) -> PassThroughEndpointLoggingTypedDict: - """ - Records model and provider for an Azure AI Speech REST call. Azure bills per audio - hour after the fact and neither the short-audio response nor the batch job carries - a billable duration this path can trust, so response_cost is recorded as 0.0 rather - than estimated. - """ try: model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) + response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(url_route, response_body) updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict **kwargs, "model": model_name, "custom_llm_provider": AZURE_SPEECH_CUSTOM_LLM_PROVIDER, - "response_cost": 0.0, + "response_cost": response_cost, } logging_obj.model_call_details.update( model=model_name, custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER, - response_cost=0.0, + response_cost=response_cost, ) standard_logging_object: Final = get_standard_logging_object_payload( diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 919de5c1088..2ccb8ad525d 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -264,6 +264,7 @@ class PassThroughEndpointLogging: azure_speech_handler_result: Final = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( httpx_response=httpx_response, + response_body=response_body, logging_obj=logging_obj, url_route=url_route, result=result, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py index 91f7bf94281..5ffb1f7785d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -1,9 +1,11 @@ +import json from datetime import datetime from unittest.mock import MagicMock import httpx import pytest +import litellm from litellm.proxy.pass_through_endpoints.llm_provider_handlers.azure_speech_passthrough_logging_handler import ( AzureSpeechPassthroughLoggingHandler, ) @@ -13,7 +15,24 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( SHORT_AUDIO_URL = "https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US" BATCH_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2/transcriptions" -TRANSCRIPT = '{"RecognitionStatus":"Success","DisplayText":"Hello world."}' +TRANSCRIPT_BODY = {"RecognitionStatus": "Success", "Offset": 5000000, "Duration": 25000000, "DisplayText": "Hello world."} +TRANSCRIPT = json.dumps(TRANSCRIPT_BODY) +TRANSCRIPT_AUDIO_SECONDS = 3.0 +PRICE_PER_SECOND = 0.5 + + +@pytest.fixture(autouse=True) +def azure_stt_price(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setitem( + litellm.model_cost, + "azure/speech/azure-stt", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": PRICE_PER_SECOND, + "output_cost_per_second": 0.0, + }, + ) def _make_response(url: str) -> httpx.Response: @@ -30,18 +49,19 @@ def _make_logging_obj() -> MagicMock: class TestAzureSpeechPassthroughHandler: @pytest.mark.parametrize( - "url_route,expected_model", + "url_route,expected_model,expected_cost", [ - (SHORT_AUDIO_URL, "azure_speech/short-audio"), - (BATCH_URL, "azure_speech/batch-transcription"), - (f"{BATCH_URL}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files", "azure_speech/batch-transcription"), + (SHORT_AUDIO_URL, "azure_speech/short-audio", TRANSCRIPT_AUDIO_SECONDS * PRICE_PER_SECOND), + (BATCH_URL, "azure_speech/batch-transcription", 0.0), + (f"{BATCH_URL}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files", "azure_speech/batch-transcription", 0.0), ], ) - def test_records_model_provider_and_zero_cost(self, url_route: str, expected_model: str): + def test_records_model_provider_and_cost(self, url_route: str, expected_model: str, expected_cost: float): logging_obj = _make_logging_obj() handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( httpx_response=_make_response(url_route), + response_body=TRANSCRIPT_BODY, logging_obj=logging_obj, url_route=url_route, result=TRANSCRIPT, @@ -54,16 +74,65 @@ class TestAzureSpeechPassthroughHandler: assert handler_result["result"] == {"response": TRANSCRIPT} assert handler_result["kwargs"]["model"] == expected_model assert handler_result["kwargs"]["custom_llm_provider"] == "azure_speech" - assert handler_result["kwargs"]["response_cost"] == 0.0 - assert handler_result["kwargs"]["standard_logging_object"]["response_cost"] == 0.0 + assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["standard_logging_object"]["response_cost"] == pytest.approx(expected_cost) assert handler_result["kwargs"]["standard_logging_object"]["model"] == expected_model assert logging_obj.model_call_details["model"] == expected_model assert logging_obj.model_call_details["custom_llm_provider"] == "azure_speech" - assert logging_obj.model_call_details["response_cost"] == 0.0 + assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) + + @pytest.mark.parametrize( + "response_body", + [ + {"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}, + {"RecognitionStatus": "InitialSilenceTimeout"}, + {"Offset": "5000000", "Duration": "25000000"}, + {}, + [], + None, + ], + ) + def test_short_audio_without_recognized_duration_logs_zero_cost( + self, response_body: dict[str, object] | list[dict[str, object]] | None + ): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL), + response_body=response_body, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["model"] == "azure_speech/short-audio" + assert handler_result["kwargs"]["response_cost"] == 0.0 + + def test_missing_price_entry_still_logs_the_row_at_zero_cost(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delitem(litellm.model_cost, "azure/speech/azure-stt") + + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL), + response_body=TRANSCRIPT_BODY, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["model"] == "azure_speech/short-audio" + assert handler_result["kwargs"]["custom_llm_provider"] == "azure_speech" + assert handler_result["kwargs"]["response_cost"] == 0.0 def test_subscription_key_never_reaches_the_logging_payload(self): handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( httpx_response=_make_response(SHORT_AUDIO_URL), + response_body=TRANSCRIPT_BODY, logging_obj=_make_logging_obj(), url_route=SHORT_AUDIO_URL, result=TRANSCRIPT, @@ -106,18 +175,18 @@ class TestNormalizeDispatch: def test_normalize_routes_to_azure_speech_handler(self): normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( httpx_response=_make_response(SHORT_AUDIO_URL), - response_body={"RecognitionStatus": "Success"}, + response_body=TRANSCRIPT_BODY, request_body={}, logging_obj=_make_logging_obj(), url_route=SHORT_AUDIO_URL, - result=TRANSCRIPT, + result="", start_time=datetime.now(), end_time=datetime.now(), cache_hit=False, custom_llm_provider="azure_speech", ) - assert normalized["standard_logging_response_object"] == {"response": TRANSCRIPT} + assert normalized["standard_logging_response_object"] == {"response": ""} assert normalized["kwargs"]["model"] == "azure_speech/short-audio" assert normalized["kwargs"]["custom_llm_provider"] == "azure_speech" - assert normalized["kwargs"]["response_cost"] == 0.0 + assert normalized["kwargs"]["response_cost"] == pytest.approx(TRANSCRIPT_AUDIO_SECONDS * PRICE_PER_SECOND) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9fe7f5b6ee9..a1102543ae8 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -6278,6 +6278,49 @@ class TestAzureSpeechProxyRoute: ("azure_speech/batch-transcription", "azure_speech", 0.0) ] + def test_short_audio_spend_is_priced_from_the_recognized_duration( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.payloads.append(kwargs["standard_logging_object"]) + + recorder: Final = _Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder]) + monkeypatch.setitem( + litellm.model_cost, + "azure/speech/azure-stt", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": 0.25, + "output_cost_per_second": 0.0, + }, + ) + transcript: Final = {**AZURE_SPEECH_TRANSCRIPT, "Offset": 10_000_000, "Duration": 30_000_000} + with respx.mock(assert_all_called=True) as upstream: + upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock( + return_value=httpx.Response(200, json=transcript) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + content=AZURE_SPEECH_WAV_BYTES, + headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 200 + assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [ + ("azure_speech/short-audio", "azure_speech") + ] + assert recorder.payloads[0]["response_cost"] == pytest.approx(4.0 * 0.25) + def test_api_base_wins_over_region_for_both_families( self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: From 6e56ba86c5ed3851f8ef4b4b309c1e85949606f9 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 04:03:22 +0000 Subject: [PATCH 3/7] test(proxy): clear leaked auth dependency override before Azure Speech real-auth tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../pass_through_endpoints/test_llm_pass_through_endpoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index a1102543ae8..05ef44b89b6 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -6423,6 +6423,7 @@ class TestAzureSpeechRawBodyThroughRealAuth: ) -> httpx.Response: from litellm.proxy.proxy_server import app + monkeypatch.delitem(app.dependency_overrides, user_api_key_auth, raising=False) monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus") monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False) From 1d8f19e4fde7615dc1828ebf7b4bf1e0d510af85 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 04:14:26 +0000 Subject: [PATCH 4/7] fix(proxy): keep Azure Speech multipart bodies intact through auth user_api_key_auth called request.form() on multipart Azure Speech batch uploads, consuming the Starlette stream before the pass-through handler could read the raw bytes. The opaque body predicate now covers multipart on the whole /azure_speech prefix so auth caches an empty parsed body and the upload is forwarded byte for byte Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/http_parsing_utils.py | 9 +-- .../test_llm_pass_through_endpoints.py | 63 ++++++++++++++++--- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 29dc36f3dba..1c17c46e5af 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -11,7 +11,6 @@ from typing_extensions import NotRequired, ReadOnly, Required from litellm._logging import verbose_proxy_logger from litellm.constants import ( AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX, - AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB, ) @@ -220,9 +219,11 @@ async def _read_request_body(request: Request | None) -> dict: def is_opaque_audio_pass_through_request(route: str, content_type: str) -> bool: - return route.startswith( - f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}{AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX}" - ) and _normalize_media_type(content_type).startswith("audio/") + """Azure Speech bodies (raw audio, multipart uploads) are forwarded byte for byte, so auth must not consume them.""" + media_type: Final = _normalize_media_type(content_type) + return route.startswith(f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/") and ( + media_type.startswith("audio/") or media_type == "multipart/form-data" + ) async def read_raw_json_body(request: Request | None) -> bytes | None: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 05ef44b89b6..43fc28c34c4 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -6418,8 +6418,8 @@ def _azure_speech_real_auth_attrs() -> dict[str, object]: class TestAzureSpeechRawBodyThroughRealAuth: """user_api_key_auth reads the body before the route runs; raw audio must not be parsed as JSON.""" - def _post_wav( - self, monkeypatch: pytest.MonkeyPatch, path: str, api_key: str, body: bytes = AZURE_SPEECH_WAV_BYTES + def _post( + self, monkeypatch: pytest.MonkeyPatch, path: str, api_key: str, content_type: str, body: bytes ) -> httpx.Response: from litellm.proxy.proxy_server import app @@ -6437,9 +6437,14 @@ class TestAzureSpeechRawBodyThroughRealAuth: path, params={"language": "en-US"}, content=body, - headers={"Content-Type": "audio/wav", "Authorization": f"Bearer {api_key}"}, + headers={"Content-Type": content_type, "Authorization": f"Bearer {api_key}"}, ) + def _post_wav( + self, monkeypatch: pytest.MonkeyPatch, path: str, api_key: str, body: bytes = AZURE_SPEECH_WAV_BYTES + ) -> httpx.Response: + return self._post(monkeypatch, path, api_key, "audio/wav", body) + @pytest.mark.parametrize("body", [AZURE_SPEECH_WAV_BYTES, AZURE_SPEECH_NON_UTF8_WAV_BYTES], ids=["ascii", "binary"]) def test_master_key_with_raw_wav_body_reaches_azure_without_a_parse_attempt( self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, body: bytes @@ -6466,11 +6471,55 @@ class TestAzureSpeechRawBodyThroughRealAuth: assert response.status_code in (400, 401), response.text assert not catch_all.called - @pytest.mark.parametrize("path", ["/v1/chat/completions", f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}"]) - def test_audio_content_type_off_the_short_audio_route_is_still_parsed_as_json( - self, monkeypatch: pytest.MonkeyPatch, path: str + def test_master_key_with_multipart_batch_upload_is_forwarded_byte_for_byte( + self, monkeypatch: pytest.MonkeyPatch ) -> None: - response = self._post_wav(monkeypatch, path, "sk-master-key", body=b'{}{"model": "gpt-4o"}') + boundary: Final = "lit7939boundary" + multipart_body: Final = ( + f"--{boundary}\r\nContent-Disposition: form-data; name=\"definition\"\r\n\r\n".encode() + + json.dumps({"locales": ["en-US"]}).encode() + + f"\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"audio\"; filename=\"eagle.wav\"\r\n" + "Content-Type: audio/wav\r\n\r\n".encode() + + AZURE_SPEECH_NON_UTF8_WAV_BYTES + + f"\r\n--{boundary}--\r\n".encode() + ) + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( + return_value=httpx.Response(201, json={"status": "NotStarted"}) + ) + + response = self._post( + monkeypatch, + f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", + "sk-master-key", + f"multipart/form-data; boundary={boundary}", + multipart_body, + ) + + assert (response.status_code, response.json()) == (201, {"status": "NotStarted"}) + sent = route.calls.last.request + assert sent.content == multipart_body + assert sent.headers["content-type"] == f"multipart/form-data; boundary={boundary}" + assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" + + @pytest.mark.parametrize("content_type", ["audio/wav", "multipart/form-data; boundary=x"]) + def test_wrong_litellm_key_with_multipart_batch_upload_is_rejected( + self, monkeypatch: pytest.MonkeyPatch, content_type: str + ) -> None: + with respx.mock(assert_all_called=False) as upstream: + catch_all = upstream.route().mock(return_value=httpx.Response(200)) + + response = self._post( + monkeypatch, f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", "sk-wrong", content_type, b"--x--\r\n" + ) + + assert response.status_code in (400, 401), response.text + assert not catch_all.called + + def test_audio_content_type_off_the_azure_speech_route_is_still_parsed_as_json( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + response = self._post_wav(monkeypatch, "/v1/chat/completions", "sk-master-key", body=b'{}{"model": "gpt-4o"}') assert response.status_code == 400 assert "Invalid JSON payload" in response.text From 5f64dfd8dd4deffdee76c673325590176fc48001 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 18:14:04 +0000 Subject: [PATCH 5/7] fix(proxy): price Azure Speech fast transcription and limit unpriced batch writes to admins Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 4 + .../llm_passthrough_endpoints.py | 22 +++ ...zure_speech_passthrough_logging_handler.py | 30 +++- ...zure_speech_passthrough_logging_handler.py | 39 ++++- .../test_llm_pass_through_endpoints.py | 146 ++++++++++++++++-- 5 files changed, 220 insertions(+), 21 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 15a1d054e26..d9da0cc0f64 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1574,13 +1574,17 @@ AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech" AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech" AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: Final = "/speech/" AZURE_SPEECH_BATCH_PATH_PREFIX: Final = "/speechtotext/" +AZURE_SPEECH_FAST_TRANSCRIPTION_PATH: Final = "/speechtotext/transcriptions:transcribe" +AZURE_SPEECH_UNPRICED_WRITE_METHODS: Final = frozenset({"POST", "PUT"}) AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com" AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com" AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key" AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio" AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription" +AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL: Final = "fast-transcription" AZURE_SPEECH_PRICING_MODEL: Final = "azure/speech/azure-stt" AZURE_SPEECH_TICKS_PER_SECOND: Final = 10_000_000 +AZURE_SPEECH_MILLISECONDS_PER_SECOND: Final = 1_000 BASE_MCP_ROUTE: Final = "/mcp" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index e64eac87a7f..b8966508f81 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -33,10 +33,12 @@ from litellm.constants import ( AZURE_SPEECH_BATCH_PATH_PREFIX, AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN, AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + AZURE_SPEECH_FAST_TRANSCRIPTION_PATH, AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX, AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, AZURE_SPEECH_STT_DOMAIN, AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, + AZURE_SPEECH_UNPRICED_WRITE_METHODS, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix @@ -65,6 +67,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( get_request_body, is_json_content_type, ) +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.common_utils.sse_keepalive import ( wrap_passthrough_sse_bytes_with_keepalive_pings, ) @@ -1357,6 +1360,14 @@ def resolve_azure_speech_base_url(endpoint_path: str, api_base: str | None, regi return httpx.URL(f"https://{region}.{domain}") +def azure_speech_write_is_unpriced(method: str, endpoint_path: str) -> bool: + return ( + endpoint_path.startswith(AZURE_SPEECH_BATCH_PATH_PREFIX) + and endpoint_path != AZURE_SPEECH_FAST_TRANSCRIPTION_PATH + and method.upper() in AZURE_SPEECH_UNPRICED_WRITE_METHODS + ) + + @router.api_route( f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list @@ -1395,6 +1406,17 @@ async def azure_speech_proxy_route( "AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE in the proxy environment." ), ) + if azure_speech_write_is_unpriced( + method=request.method, endpoint_path=normalized_endpoint_path + ) and not is_proxy_admin(user_api_key_dict): + raise HTTPException( + status_code=403, + detail=( + f"{request.method} {normalized_endpoint_path} creates Azure Speech work whose cost is unknown at " + "request time, so it is limited to proxy admin keys. Use " + f"{AZURE_SPEECH_FAST_TRANSCRIPTION_PATH} for transcription that is priced per request." + ), + ) azure_speech_api_key: Final = passthrough_endpoint_router.get_credentials( custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER, region_name=None, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py index 74587acd453..588b7cc8e56 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -9,6 +9,9 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( AZURE_SPEECH_BATCH_MODEL, AZURE_SPEECH_CUSTOM_LLM_PROVIDER, + AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL, + AZURE_SPEECH_FAST_TRANSCRIPTION_PATH, + AZURE_SPEECH_MILLISECONDS_PER_SECOND, AZURE_SPEECH_PRICING_MODEL, AZURE_SPEECH_SHORT_AUDIO_MODEL, AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, @@ -28,10 +31,16 @@ class AzureSpeechPassthroughLoggingHandler: def _is_short_audio_route(url_route: str) -> bool: return urlparse(url_route).path.startswith(AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX) + @staticmethod + def _is_fast_transcription_route(url_route: str) -> bool: + return urlparse(url_route).path.endswith(AZURE_SPEECH_FAST_TRANSCRIPTION_PATH) + @staticmethod def _model_from_url_route(url_route: str) -> str: if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_SHORT_AUDIO_MODEL}" + if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route): + return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL}" return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}" @staticmethod @@ -45,10 +54,25 @@ class AzureSpeechPassthroughLoggingHandler: return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND @staticmethod - def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: - if not AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): + def _fast_transcription_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float: + if not isinstance(response_body, Mapping): return 0.0 - audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body) + duration_milliseconds: Final = response_body.get("durationMilliseconds") + if not isinstance(duration_milliseconds, int): + return 0.0 + return duration_milliseconds / AZURE_SPEECH_MILLISECONDS_PER_SECOND + + @staticmethod + def _billed_audio_seconds(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: + if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): + return AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body) + if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route): + return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body) + return 0.0 + + @staticmethod + def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: + audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(url_route, response_body) if audio_seconds <= 0.0: return 0.0 try: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py index 5ffb1f7785d..50d3de64f72 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -13,9 +13,19 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) -SHORT_AUDIO_URL = "https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US" +SHORT_AUDIO_URL = ( + "https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US" +) BATCH_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2/transcriptions" -TRANSCRIPT_BODY = {"RecognitionStatus": "Success", "Offset": 5000000, "Duration": 25000000, "DisplayText": "Hello world."} +FAST_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/transcriptions:transcribe?api-version=2024-11-15" +FAST_BODY = {"durationMilliseconds": 5061, "combinedPhrases": [{"text": "Hello world."}]} +FAST_AUDIO_SECONDS = 5.061 +TRANSCRIPT_BODY = { + "RecognitionStatus": "Success", + "Offset": 5000000, + "Duration": 25000000, + "DisplayText": "Hello world.", +} TRANSCRIPT = json.dumps(TRANSCRIPT_BODY) TRANSCRIPT_AUDIO_SECONDS = 3.0 PRICE_PER_SECOND = 0.5 @@ -52,6 +62,7 @@ class TestAzureSpeechPassthroughHandler: "url_route,expected_model,expected_cost", [ (SHORT_AUDIO_URL, "azure_speech/short-audio", TRANSCRIPT_AUDIO_SECONDS * PRICE_PER_SECOND), + (FAST_URL, "azure_speech/fast-transcription", FAST_AUDIO_SECONDS * PRICE_PER_SECOND), (BATCH_URL, "azure_speech/batch-transcription", 0.0), (f"{BATCH_URL}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files", "azure_speech/batch-transcription", 0.0), ], @@ -61,7 +72,7 @@ class TestAzureSpeechPassthroughHandler: handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( httpx_response=_make_response(url_route), - response_body=TRANSCRIPT_BODY, + response_body={**TRANSCRIPT_BODY, **FAST_BODY}, logging_obj=logging_obj, url_route=url_route, result=TRANSCRIPT, @@ -110,6 +121,28 @@ class TestAzureSpeechPassthroughHandler: assert handler_result["kwargs"]["model"] == "azure_speech/short-audio" assert handler_result["kwargs"]["response_cost"] == 0.0 + @pytest.mark.parametrize( + "response_body", + [{"durationMilliseconds": 0}, {"durationMilliseconds": "5061"}, {"duration": 5061}, {}, [], None], + ) + def test_fast_transcription_without_duration_milliseconds_logs_zero_cost( + self, response_body: dict[str, object] | list[dict[str, object]] | None + ): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(FAST_URL), + response_body=response_body, + logging_obj=_make_logging_obj(), + url_route=FAST_URL, + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription" + assert handler_result["kwargs"]["response_cost"] == 0.0 + def test_missing_price_entry_still_logs_the_row_at_zero_cost(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.delitem(litellm.model_cost, "azure/speech/azure-stt") diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 43fc28c34c4..eb0c607fbef 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -6141,6 +6141,7 @@ class TestAzureRelayDeploymentSegment: AZURE_SPEECH_SHORT_AUDIO_ENDPOINT: Final = "/speech/recognition/conversation/cognitiveservices/v1" AZURE_SPEECH_BATCH_ENDPOINT: Final = "/speechtotext/v3.2/transcriptions" +AZURE_SPEECH_FAST_ENDPOINT: Final = "/speechtotext/transcriptions:transcribe" AZURE_SPEECH_PCM16_HEADER: Final = ( b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00" ) @@ -6149,8 +6150,7 @@ AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."} -@pytest.fixture -def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: +def _azure_speech_test_client(monkeypatch: pytest.MonkeyPatch, caller: UserAPIKeyAuth) -> TestClient: from litellm.proxy.proxy_server import app monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") @@ -6159,8 +6159,20 @@ def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient] monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() - monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) - yield TestClient(app) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: caller) + return TestClient(app) + + +@pytest.fixture +def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + yield _azure_speech_test_client(monkeypatch, UserAPIKeyAuth(api_key="sk-virtual")) + + +@pytest.fixture +def azure_speech_admin_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + yield _azure_speech_test_client( + monkeypatch, UserAPIKeyAuth(api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + ) class TestAzureSpeechProxyRoute: @@ -6193,17 +6205,19 @@ class TestAzureSpeechProxyRoute: assert "authorization" not in sent.headers assert "caller-supplied-key" not in repr(sent.headers) - def test_batch_json_goes_to_the_cognitive_services_host(self, azure_speech_client: TestClient) -> None: + def test_admin_batch_job_creation_goes_to_the_cognitive_services_host( + self, azure_speech_admin_client: TestClient + ) -> None: body: Final = {"contentUrls": ["https://example.com/a.wav"], "locale": "en-US", "displayName": "job"} with respx.mock(assert_all_called=True) as upstream: route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( return_value=httpx.Response(201, json={"self": "https://eastus.api.cognitive.microsoft.com/x"}) ) - response = azure_speech_client.post( + response = azure_speech_admin_client.post( f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", json=body, - headers={"Authorization": "Bearer sk-virtual"}, + headers={"Authorization": "Bearer sk-admin"}, ) assert response.status_code == 201 @@ -6212,22 +6226,80 @@ class TestAzureSpeechProxyRoute: assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" assert "authorization" not in sent.headers - def test_batch_multipart_upload_is_forwarded_byte_for_byte(self, azure_speech_client: TestClient) -> None: + @pytest.mark.parametrize( + "method,endpoint", + [ + ("POST", AZURE_SPEECH_BATCH_ENDPOINT), + ("POST", "/speechtotext/v3.2/models"), + ("PUT", "/speechtotext/v3.2/endpoints/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"), + ], + ) + def test_non_admin_key_cannot_create_unpriced_batch_work( + self, azure_speech_client: TestClient, method: str, endpoint: str + ) -> None: + with respx.mock(assert_all_called=False) as upstream: + catch_all = upstream.route().mock(return_value=httpx.Response(201, json={"status": "NotStarted"})) + + response = azure_speech_client.request( + method, + f"/azure_speech{endpoint}", + json={"contentUrls": ["https://example.com/a.wav"], "locale": "en-US"}, + headers={"Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 403, response.text + assert AZURE_SPEECH_FAST_ENDPOINT in response.text + assert not catch_all.called + + def test_non_admin_key_can_still_read_delete_and_fast_transcribe_in_the_batch_family( + self, azure_speech_client: TestClient + ) -> None: + job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab" with respx.mock(assert_all_called=True) as upstream: - route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( - return_value=httpx.Response(201, json={"status": "NotStarted"}) + upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock( + return_value=httpx.Response(200, json={"status": "Succeeded"}) + ) + upstream.delete(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock( + return_value=httpx.Response(204) + ) + upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []}) + ) + + statuses = [ + azure_speech_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}), + azure_speech_client.delete(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}), + azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", + params={"api-version": "2024-11-15"}, + files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")}, + data={"definition": json.dumps({"locales": ["en-US"]})}, + headers={"Authorization": "Bearer sk-virtual"}, + ), + ] + + assert [r.status_code for r in statuses] == [200, 204, 200] + + def test_fast_transcription_multipart_upload_is_forwarded_byte_for_byte( + self, azure_speech_client: TestClient + ) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []}) ) response = azure_speech_client.post( - f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", + f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", + params={"api-version": "2024-11-15"}, files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")}, data={"definition": json.dumps({"locales": ["en-US"]})}, headers={"Authorization": "Bearer sk-virtual"}, ) - assert response.status_code == 201 + assert response.status_code == 200 sent = route.calls.last.request assert sent.headers["content-type"].startswith("multipart/form-data; boundary=") + assert dict(sent.url.params) == {"api-version": "2024-11-15"} assert AZURE_SPEECH_NON_UTF8_WAV_BYTES in sent.content assert b'name="definition"' in sent.content assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" @@ -6247,7 +6319,7 @@ class TestAzureSpeechProxyRoute: @pytest.mark.parametrize("method", ["GET", "POST"]) def test_batch_requests_are_logged_as_azure_speech_not_assemblyai( - self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch, method: str + self, azure_speech_admin_client: TestClient, monkeypatch: pytest.MonkeyPatch, method: str ) -> None: from litellm.integrations.custom_logger import CustomLogger @@ -6266,11 +6338,11 @@ class TestAzureSpeechProxyRoute: return_value=httpx.Response(200, json={"values": []}) ) - response = azure_speech_client.request( + response = azure_speech_admin_client.request( method, f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", json={"locale": "en-US"} if method == "POST" else None, - headers={"Authorization": "Bearer sk-virtual"}, + headers={"Authorization": "Bearer sk-admin"}, ) assert response.status_code == 200 @@ -6278,6 +6350,50 @@ class TestAzureSpeechProxyRoute: ("azure_speech/batch-transcription", "azure_speech", 0.0) ] + def test_fast_transcription_spend_is_priced_from_duration_milliseconds( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.payloads.append(kwargs["standard_logging_object"]) + + recorder: Final = _Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder]) + monkeypatch.setitem( + litellm.model_cost, + "azure/speech/azure-stt", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": 0.25, + "output_cost_per_second": 0.0, + }, + ) + with respx.mock(assert_all_called=True) as upstream: + upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []}) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", + params={"api-version": "2024-11-15"}, + files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")}, + data={"definition": json.dumps({"locales": ["en-US"]})}, + headers={"Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 200 + assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [ + ("azure_speech/fast-transcription", "azure_speech") + ] + assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * 0.25) + def test_short_audio_spend_is_priced_from_the_recognized_duration( self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: From 25846d13417af204bb52154c125654dcc4da6411 Mon Sep 17 00:00:00 2001 From: yassin Date: Thu, 17 Sep 2026 20:29:51 +0000 Subject: [PATCH 6/7] fix(proxy): limit the whole Azure Speech batch API to proxy admin keys Ordinary keys could read, patch and delete batch transcription jobs that other keys created with the proxy's shared Azure subscription, so every /speechtotext/v3.2 method is now admin only while fast transcription stays open. Also clears SERVER_ROOT_PATH in the real-auth test helper because test_custom_proxy leaves it set at import time and the shared app then 404s pass-through routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 - .../llm_passthrough_endpoints.py | 15 ++--- .../test_llm_pass_through_endpoints.py | 65 ++++++++++++------- 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index d9da0cc0f64..62523c5d2c3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1575,7 +1575,6 @@ AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech" AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: Final = "/speech/" AZURE_SPEECH_BATCH_PATH_PREFIX: Final = "/speechtotext/" AZURE_SPEECH_FAST_TRANSCRIPTION_PATH: Final = "/speechtotext/transcriptions:transcribe" -AZURE_SPEECH_UNPRICED_WRITE_METHODS: Final = frozenset({"POST", "PUT"}) AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com" AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com" AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index b8966508f81..8d18de42f45 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -38,7 +38,6 @@ from litellm.constants import ( AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX, AZURE_SPEECH_STT_DOMAIN, AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, - AZURE_SPEECH_UNPRICED_WRITE_METHODS, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix @@ -1360,11 +1359,10 @@ def resolve_azure_speech_base_url(endpoint_path: str, api_base: str | None, regi return httpx.URL(f"https://{region}.{domain}") -def azure_speech_write_is_unpriced(method: str, endpoint_path: str) -> bool: +def azure_speech_path_manages_shared_resources(endpoint_path: str) -> bool: return ( endpoint_path.startswith(AZURE_SPEECH_BATCH_PATH_PREFIX) and endpoint_path != AZURE_SPEECH_FAST_TRANSCRIPTION_PATH - and method.upper() in AZURE_SPEECH_UNPRICED_WRITE_METHODS ) @@ -1406,15 +1404,14 @@ async def azure_speech_proxy_route( "AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE in the proxy environment." ), ) - if azure_speech_write_is_unpriced( - method=request.method, endpoint_path=normalized_endpoint_path - ) and not is_proxy_admin(user_api_key_dict): + if azure_speech_path_manages_shared_resources(normalized_endpoint_path) and not is_proxy_admin(user_api_key_dict): raise HTTPException( status_code=403, detail=( - f"{request.method} {normalized_endpoint_path} creates Azure Speech work whose cost is unknown at " - "request time, so it is limited to proxy admin keys. Use " - f"{AZURE_SPEECH_FAST_TRANSCRIPTION_PATH} for transcription that is priced per request." + f"{request.method} {normalized_endpoint_path} manages batch transcription resources that belong to " + "the proxy's Azure Speech subscription and whose cost is unknown at request time, so it is limited " + f"to proxy admin keys. Use {AZURE_SPEECH_FAST_TRANSCRIPTION_PATH} for transcription that is priced " + "per request." ), ) azure_speech_api_key: Final = passthrough_endpoint_router.get_credentials( diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index eb0c607fbef..0e8a2b9e0e9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -6232,13 +6232,17 @@ class TestAzureSpeechProxyRoute: ("POST", AZURE_SPEECH_BATCH_ENDPOINT), ("POST", "/speechtotext/v3.2/models"), ("PUT", "/speechtotext/v3.2/endpoints/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"), + ("GET", AZURE_SPEECH_BATCH_ENDPOINT), + ("GET", f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files"), + ("PATCH", f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"), + ("DELETE", f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"), ], ) - def test_non_admin_key_cannot_create_unpriced_batch_work( + def test_non_admin_key_cannot_manage_shared_batch_resources( self, azure_speech_client: TestClient, method: str, endpoint: str ) -> None: with respx.mock(assert_all_called=False) as upstream: - catch_all = upstream.route().mock(return_value=httpx.Response(201, json={"status": "NotStarted"})) + catch_all = upstream.route().mock(return_value=httpx.Response(200, json={"status": "Succeeded"})) response = azure_speech_client.request( method, @@ -6251,9 +6255,23 @@ class TestAzureSpeechProxyRoute: assert AZURE_SPEECH_FAST_ENDPOINT in response.text assert not catch_all.called - def test_non_admin_key_can_still_read_delete_and_fast_transcribe_in_the_batch_family( - self, azure_speech_client: TestClient - ) -> None: + def test_non_admin_key_can_still_fast_transcribe_in_the_batch_family(self, azure_speech_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []}) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", + params={"api-version": "2024-11-15"}, + files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")}, + data={"definition": json.dumps({"locales": ["en-US"]})}, + headers={"Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 200 + + def test_admin_key_reads_and_deletes_batch_jobs(self, azure_speech_admin_client: TestClient) -> None: job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab" with respx.mock(assert_all_called=True) as upstream: upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock( @@ -6262,23 +6280,15 @@ class TestAzureSpeechProxyRoute: upstream.delete(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock( return_value=httpx.Response(204) ) - upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( - return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []}) - ) statuses = [ - azure_speech_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}), - azure_speech_client.delete(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}), - azure_speech_client.post( - f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", - params={"api-version": "2024-11-15"}, - files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")}, - data={"definition": json.dumps({"locales": ["en-US"]})}, - headers={"Authorization": "Bearer sk-virtual"}, + azure_speech_admin_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-admin"}), + azure_speech_admin_client.delete( + f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-admin"} ), ] - assert [r.status_code for r in statuses] == [200, 204, 200] + assert [r.status_code for r in statuses] == [200, 204] def test_fast_transcription_multipart_upload_is_forwarded_byte_for_byte( self, azure_speech_client: TestClient @@ -6305,14 +6315,16 @@ class TestAzureSpeechProxyRoute: assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" assert "authorization" not in sent.headers - def test_batch_get_is_forwarded_with_the_job_id_path(self, azure_speech_client: TestClient) -> None: + def test_batch_get_is_forwarded_with_the_job_id_path(self, azure_speech_admin_client: TestClient) -> None: job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files" with respx.mock(assert_all_called=True) as upstream: route = upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock( return_value=httpx.Response(200, json={"values": []}) ) - response = azure_speech_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}) + response = azure_speech_admin_client.get( + f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-admin"} + ) assert (response.status_code, response.json()) == (200, {"values": []}) assert route.calls.last.request.headers["ocp-apim-subscription-key"] == "server-subscription-key" @@ -6445,8 +6457,8 @@ class TestAzureSpeechProxyRoute: short_audio = upstream.post( f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}" ).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT)) - batch = upstream.get(f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock( - return_value=httpx.Response(200, json={"values": []}) + fast = upstream.post(f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []}) ) azure_speech_client.post( @@ -6454,9 +6466,15 @@ class TestAzureSpeechProxyRoute: content=AZURE_SPEECH_WAV_BYTES, headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, ) - azure_speech_client.get(f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", headers={"Authorization": "Bearer x"}) + azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", + params={"api-version": "2024-11-15"}, + files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")}, + data={"definition": json.dumps({"locales": ["en-US"]})}, + headers={"Authorization": "Bearer sk-virtual"}, + ) - assert short_audio.called and batch.called + assert short_audio.called and fast.called @pytest.mark.parametrize("endpoint", ["openai/deployments/whisper/audio/transcriptions", "speech", "speechtotext"]) def test_unknown_path_family_is_rejected_before_any_upstream_call( @@ -6543,6 +6561,7 @@ class TestAzureSpeechRawBodyThroughRealAuth: monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus") monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() with patch.multiple( # test-quality-ok: the real user_api_key_auth reads proxy_server module globals (master_key, caches) that have no injection seam From 1e7c5400fd792c2b7f5d9915ac832c4147f256dc Mon Sep 17 00:00:00 2001 From: yassin Date: Fri, 18 Sep 2026 19:27:25 +0000 Subject: [PATCH 7/7] fix(proxy): canonicalize azure speech paths and bill uploaded short audio Resolve dot segments in the /azure_speech endpoint path before the endpoint family and the admin-only batch guard are decided, so the guard and the forwarded upstream path agree. Bill short-audio requests for the longer of the uploaded audio duration and the recognized duration, so a NoMatch or silence response still charges for the audio Azure processed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_passthrough_endpoints.py | 16 ++- ...zure_speech_passthrough_logging_handler.py | 40 +++++++- ...zure_speech_passthrough_logging_handler.py | 99 ++++++++++++++++--- .../test_llm_pass_through_endpoints.py | 75 ++++++++++++++ 4 files changed, 207 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index ab4b636cb60..ed70369506d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -12,6 +12,7 @@ import hmac import inspect import json import os +import posixpath import re from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass @@ -1408,6 +1409,18 @@ def azure_speech_path_manages_shared_resources(endpoint_path: str) -> bool: ) +def canonical_azure_speech_endpoint_path(endpoint: str) -> str: + """ + The path Azure will actually serve, with ``.`` and ``..`` segments resolved, so the + endpoint family and the admin guard are decided on the same path the upstream request uses. + """ + raw_path: Final = httpx.URL(endpoint).path + resolved_path: Final = posixpath.normpath(f"/{raw_path.lstrip('/')}") + if raw_path.endswith("/") and resolved_path != "/": + return f"{resolved_path}/" + return resolved_path + + @router.api_route( f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list @@ -1430,8 +1443,7 @@ async def azure_speech_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) """ - endpoint_path: Final = httpx.URL(endpoint).path - normalized_endpoint_path: Final = endpoint_path if endpoint_path.startswith("/") else f"/{endpoint_path}" + normalized_endpoint_path: Final = canonical_azure_speech_endpoint_path(endpoint) base_url: Final = resolve_azure_speech_base_url( endpoint_path=normalized_endpoint_path, api_base=get_secret_str(secret_name="AZURE_SPEECH_API_BASE"), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py index 588b7cc8e56..8cd1b137de8 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -18,6 +18,7 @@ from litellm.constants import ( AZURE_SPEECH_TICKS_PER_SECOND, ) from litellm.cost_calculator import transcription_cost +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -53,6 +54,23 @@ class AzureSpeechPassthroughLoggingHandler: return 0.0 return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND + @staticmethod + def _uploaded_audio_seconds(httpx_response: httpx.Response) -> float: + try: + uploaded_audio: Final = httpx_response.request.content + except RuntimeError: + return 0.0 + return calculate_request_duration(uploaded_audio) or 0.0 + + @staticmethod + def _short_audio_seconds( + httpx_response: httpx.Response, response_body: Mapping[str, object] | Sequence[object] | None + ) -> float: + return max( + AzureSpeechPassthroughLoggingHandler._uploaded_audio_seconds(httpx_response), + AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body), + ) + @staticmethod def _fast_transcription_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float: if not isinstance(response_body, Mapping): @@ -63,16 +81,26 @@ class AzureSpeechPassthroughLoggingHandler: return duration_milliseconds / AZURE_SPEECH_MILLISECONDS_PER_SECOND @staticmethod - def _billed_audio_seconds(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: + def _billed_audio_seconds( + url_route: str, + httpx_response: httpx.Response, + response_body: Mapping[str, object] | Sequence[object] | None, + ) -> float: if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): - return AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body) + return AzureSpeechPassthroughLoggingHandler._short_audio_seconds(httpx_response, response_body) if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route): return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body) return 0.0 @staticmethod - def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: - audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(url_route, response_body) + def _response_cost( + url_route: str, + httpx_response: httpx.Response, + response_body: Mapping[str, object] | Sequence[object] | None, + ) -> float: + audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds( + url_route, httpx_response, response_body + ) if audio_seconds <= 0.0: return 0.0 try: @@ -103,7 +131,9 @@ class AzureSpeechPassthroughLoggingHandler: ) -> PassThroughEndpointLoggingTypedDict: try: model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) - response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(url_route, response_body) + response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost( + url_route, httpx_response, response_body + ) updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict **kwargs, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py index 50d3de64f72..670f8e65823 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -1,5 +1,8 @@ +import io import json +import wave from datetime import datetime +from typing import Final from unittest.mock import MagicMock import httpx @@ -29,6 +32,25 @@ TRANSCRIPT_BODY = { TRANSCRIPT = json.dumps(TRANSCRIPT_BODY) TRANSCRIPT_AUDIO_SECONDS = 3.0 PRICE_PER_SECOND = 0.5 +WAV_SAMPLE_RATE: Final = 16000 +UNRECOGNIZED_BODIES: Final = ( + {"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}, + {"RecognitionStatus": "InitialSilenceTimeout"}, + {"Offset": "5000000", "Duration": "25000000"}, + {}, + [], + None, +) + + +def _pcm16_wav(seconds: float) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(WAV_SAMPLE_RATE) + wav.writeframes(b"\x00\x00" * int(seconds * WAV_SAMPLE_RATE)) + return buffer.getvalue() @pytest.fixture(autouse=True) @@ -45,8 +67,8 @@ def azure_stt_price(monkeypatch: pytest.MonkeyPatch): ) -def _make_response(url: str) -> httpx.Response: - request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}) +def _make_response(url: str, uploaded: bytes = b"") -> httpx.Response: + request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}, content=uploaded) return httpx.Response(200, request=request, text=TRANSCRIPT) @@ -92,22 +114,13 @@ class TestAzureSpeechPassthroughHandler: assert logging_obj.model_call_details["custom_llm_provider"] == "azure_speech" assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) - @pytest.mark.parametrize( - "response_body", - [ - {"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}, - {"RecognitionStatus": "InitialSilenceTimeout"}, - {"Offset": "5000000", "Duration": "25000000"}, - {}, - [], - None, - ], - ) - def test_short_audio_without_recognized_duration_logs_zero_cost( - self, response_body: dict[str, object] | list[dict[str, object]] | None + @pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES) + @pytest.mark.parametrize("uploaded", [b"", b"not audio at all"]) + def test_short_audio_with_neither_recognized_nor_decodable_audio_logs_zero_cost( + self, response_body: dict[str, object] | list[dict[str, object]] | None, uploaded: bytes ): handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( - httpx_response=_make_response(SHORT_AUDIO_URL), + httpx_response=_make_response(SHORT_AUDIO_URL, uploaded), response_body=response_body, logging_obj=_make_logging_obj(), url_route=SHORT_AUDIO_URL, @@ -121,6 +134,60 @@ class TestAzureSpeechPassthroughHandler: assert handler_result["kwargs"]["model"] == "azure_speech/short-audio" assert handler_result["kwargs"]["response_cost"] == 0.0 + @pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES) + def test_short_audio_bills_the_uploaded_audio_when_nothing_was_recognized( + self, response_body: dict[str, object] | list[dict[str, object]] | None + ): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=2.0)), + response_body=response_body, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(2.0 * PRICE_PER_SECOND) + + @pytest.mark.parametrize( + "uploaded_seconds,expected_seconds", + [(1.0, TRANSCRIPT_AUDIO_SECONDS), (TRANSCRIPT_AUDIO_SECONDS + 2.0, TRANSCRIPT_AUDIO_SECONDS + 2.0)], + ) + def test_short_audio_bills_the_longer_of_uploaded_and_recognized_audio( + self, uploaded_seconds: float, expected_seconds: float + ): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=uploaded_seconds)), + response_body=TRANSCRIPT_BODY, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_seconds * PRICE_PER_SECOND) + + def test_fast_transcription_ignores_the_uploaded_multipart_body(self): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(FAST_URL, _pcm16_wav(seconds=30.0)), + response_body=FAST_BODY, + logging_obj=_make_logging_obj(), + url_route=FAST_URL, + result=json.dumps(FAST_BODY), + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND) + @pytest.mark.parametrize( "response_body", [{"durationMilliseconds": 0}, {"durationMilliseconds": "5061"}, {"duration": 5061}, {}, [], None], diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9d210ac2f4f..636980eb6e3 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _proxy_general_settings, anthropic_proxy_route, azure_proxy_route, + azure_speech_proxy_route, bedrock_llm_proxy_route, bedrock_proxy_route, create_pass_through_route, @@ -6443,6 +6444,7 @@ AZURE_SPEECH_PCM16_HEADER: Final = ( b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00" ) AZURE_SPEECH_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + b"\x00" * 3072 +AZURE_SPEECH_WAV_SECONDS: Final = 3072 / (16000 * 2) AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range(256)) * 12 AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."} @@ -6826,6 +6828,79 @@ class TestAzureSpeechProxyRoute: assert "/azure_speech" in LiteLLMRoutes.mapped_pass_through_routes.value + def test_short_audio_with_no_recognized_speech_is_billed_for_the_uploaded_audio( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.payloads.append(kwargs["standard_logging_object"]) + + recorder: Final = _Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder]) + monkeypatch.setitem( + litellm.model_cost, + "azure/speech/azure-stt", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": 0.25, + "output_cost_per_second": 0.0, + }, + ) + with respx.mock(assert_all_called=True) as upstream: + upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + content=AZURE_SPEECH_WAV_BYTES, + headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 200 + assert [p["model"] for p in recorder.payloads] == ["azure_speech/short-audio"] + assert recorder.payloads[0]["response_cost"] == pytest.approx(AZURE_SPEECH_WAV_SECONDS * 0.25) + + +class TestAzureSpeechProxyRoutePathTraversal: + """Calls the route function directly because httpx clients resolve dot segments before sending.""" + + @pytest.mark.parametrize( + "endpoint", + [ + f"speech/..{AZURE_SPEECH_BATCH_ENDPOINT}", + f"speech/recognition/../..{AZURE_SPEECH_BATCH_ENDPOINT}/", + f"speech/./..{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab", + ], + ) + @pytest.mark.asyncio + async def test_dot_segments_cannot_reach_shared_batch_resources_with_a_non_admin_key( + self, monkeypatch: pytest.MonkeyPatch, endpoint: str + ) -> None: + monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") + monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus") + monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False) + request: Final = MagicMock(spec=Request) + request.method = "GET" + + with pytest.raises(HTTPException) as denied: + await azure_speech_proxy_route( + endpoint=endpoint, + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-virtual"), + ) + + assert denied.value.status_code == 403 + assert AZURE_SPEECH_FAST_ENDPOINT in str(denied.value.detail) + def _azure_speech_real_auth_attrs() -> dict[str, object]: from litellm.caching.caching import DualCache