scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 117

Merge pull request #41557 from BerriAI/litellm_azure_speech_passthrough

Browse source 
feat(proxy): add Azure AI Speech pass-through route
This commit is contained in:
Yassin Kortam 2026-09-18 14:05:55 -07:00 • committed by GitHub
commit 6759f28e73
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 1773 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
gateway/routes/allowlist.py
View file

@ -82,6 +82,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/anthropic/",
"/azure/",
"/azure_ai/",
"/azure_speech/",
"/aws/",
"/bedrock/",
"/comprehendmedical",

2
helm/litellm/templates/ingress.yaml
View file

@ -66,7 +66,7 @@
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
"/v1beta" "/interactions"
"/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/comprehendmedical" "/transcribe" "/cohere" "/gemini" "/google"
"/anthropic" "/azure" "/azure_ai" "/azure_speech" "/aws" "/bedrock" "/comprehendmedical" "/transcribe" "/cohere" "/gemini" "/google"
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
"/toolset"

15
litellm/constants.py
View file

@ -1576,6 +1576,21 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = {
# Works for all LLM pass-through endpoints (Vertex AI, Anthropic, Bedrock, etc.)
PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-"
AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech"
AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech"
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: Final = "/speech/"
AZURE_SPEECH_BATCH_PATH_PREFIX: Final = "/speechtotext/"
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH: Final = "/speechtotext/transcriptions:transcribe"
AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com"
AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com"
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key"
AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio"
AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription"
AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL: Final = "fast-transcription"
AZURE_SPEECH_PRICING_MODEL: Final = "azure/speech/azure-stt"
AZURE_SPEECH_TICKS_PER_SECOND: Final = 10_000_000
AZURE_SPEECH_MILLISECONDS_PER_SECOND: Final = 1_000
BASE_MCP_ROUTE: Final = "/mcp"
TRANSCRIBE_JOB_POLLING_INTERVAL_SECONDS: Final = 10.0

1
litellm/passthrough/utils.py
View file

@ -17,6 +17,7 @@ _PASS_THROUGH_PROTECTED_HEADERS: Final[frozenset] = frozenset(
"api-key",
"x-api-key",
"x-goog-api-key",
"ocp-apim-subscription-key",
"host",
"content-length",
"accept-encoding",

1
litellm/proxy/_lazy_features.py
View file

@ -196,6 +196,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/assemblyai/",
"/azure/",
"/azure_ai/",
"/azure_speech/",
"/bedrock/",
"/cohere/",
"/comprehendmedical",

222
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -17181,6 +17181,228 @@
]
}
},
"/azure_speech/{endpoint}": {
"delete": {
"description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)",
"operationId": "azure_speech_proxy_route_azure_speech__endpoint__delete",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Azure Speech Proxy Route",
"tags": [
"llm_passthrough"
]
},
"get": {
"description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)",
"operationId": "azure_speech_proxy_route_azure_speech__endpoint__get",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Azure Speech Proxy Route",
"tags": [
"llm_passthrough"
]
},
"patch": {
"description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)",
"operationId": "azure_speech_proxy_route_azure_speech__endpoint__patch",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Azure Speech Proxy Route",
"tags": [
"llm_passthrough"
]
},
"post": {
"description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)",
"operationId": "azure_speech_proxy_route_azure_speech__endpoint__post",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Azure Speech Proxy Route",
"tags": [
"llm_passthrough"
]
},
"put": {
"description": "Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.\n`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`\nwith the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.\n\nThe body is forwarded byte for byte and the proxy injects its own\n`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key\nand is never forwarded.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)",
"operationId": "azure_speech_proxy_route_azure_speech__endpoint__put",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Azure Speech Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/bedrock/{endpoint}": {
"delete": {
"description": "This is the v1 passthrough for Bedrock.\nV2 is handled by the `/bedrock/v2` endpoint.\n[Docs](https://docs.litellm.ai/docs/pass_through/bedrock)",

1
litellm/proxy/_types.py
View file

@ -469,6 +469,7 @@ class LiteLLMRoutes(enum.Enum):
mapped_pass_through_routes = [
"/bedrock",
"/comprehendmedical",
"/azure_speech",
"/transcribe",
"/vertex-ai",
"/vertex_ai",

7
litellm/proxy/auth/user_api_key_auth.py
View file

@ -107,6 +107,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
_safe_get_request_headers,
_safe_get_request_query_params,
_safe_set_request_parsed_body,
is_opaque_audio_pass_through_request,
populate_request_with_path_params,
read_raw_json_body,
rewrite_request_model,
@ -1356,6 +1357,12 @@ async def _read_request_body_deferring_parse_failure(
must run (resolving identity onto the request's trace) before the 400 goes
out; the caller re-raises the returned exception once identity is seeded.
"""
if is_opaque_audio_pass_through_request(
route=get_request_route(request=request),
content_type=_safe_get_request_headers(request=request).get("content-type", ""),
):
_safe_set_request_parsed_body(request=request, parsed_body={}) # mutable-ok: the body cache stores a plain dict
return {}, None # mutable-ok: request_data is a plain dict across the whole auth path
try:
parsed_body: Final = await _read_request_body(request=request)
except ProxyException as parse_exception:

14
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -9,7 +9,11 @@ from fastapi import Request, UploadFile, status
from typing_extensions import NotRequired, ReadOnly, Required
from litellm._logging import verbose_proxy_logger
from litellm.constants import CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
from litellm.constants import (
AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX,
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB,
)
from litellm.proxy._types import ProxyException
from litellm.proxy.common_utils.callback_utils import (
get_metadata_variable_name_from_kwargs,
@ -214,6 +218,14 @@ async def _read_request_body(request: Request | None) -> dict:
return {}
def is_opaque_audio_pass_through_request(route: str, content_type: str) -> bool:
"""Azure Speech bodies (raw audio, multipart uploads) are forwarded byte for byte, so auth must not consume them."""
media_type: Final = _normalize_media_type(content_type)
return route.startswith(f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/") and (
media_type.startswith("audio/") or media_type == "multipart/form-data"
)
async def read_raw_json_body(request: Request | None) -> bytes | None:
if request is None or _safe_get_request_parsed_body(request=request) is None:
return None

149
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -12,6 +12,7 @@ import hmac
import inspect
import json
import os
import posixpath
import re
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
from dataclasses import dataclass
@ -30,6 +31,14 @@ from litellm import get_llm_provider
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS,
AZURE_SPEECH_BATCH_PATH_PREFIX,
AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN,
AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH,
AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX,
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX,
AZURE_SPEECH_STT_DOMAIN,
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER,
BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES,
)
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
@ -58,6 +67,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
get_request_body,
is_json_content_type,
)
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
from litellm.proxy.common_utils.sse_keepalive import (
wrap_passthrough_sse_bytes_with_keepalive_pings,
)
@ -1358,6 +1368,145 @@ async def comprehend_medical_sdk_proxy_route(
)
AZURE_SPEECH_FORWARDED_REQUEST_HEADERS: Final = ("content-type", "accept")
AZURE_SPEECH_ENDPOINT_FAMILY_DOMAINS: Final = MappingProxyType(
{
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: AZURE_SPEECH_STT_DOMAIN,
AZURE_SPEECH_BATCH_PATH_PREFIX: AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN,
}
)
def resolve_azure_speech_base_url(endpoint_path: str, api_base: str | None, region: str | None) -> httpx.URL | None:
"""
Azure AI Speech serves the two REST families from different regional hosts: short-audio
recognition under ``{region}.stt.speech.microsoft.com`` and batch transcription under
``{region}.api.cognitive.microsoft.com``. An operator-configured ``api_base`` (custom
domain or private endpoint) serves both and wins over the region. Returns ``None`` when
the path is outside both families so the operator key is never sent for an unknown API.
"""
domain: Final = next(
(
family_domain
for family_prefix, family_domain in AZURE_SPEECH_ENDPOINT_FAMILY_DOMAINS.items()
if endpoint_path.startswith(family_prefix)
),
None,
)
if domain is None:
return None
if api_base:
return httpx.URL(api_base)
if not region:
return None
return httpx.URL(f"https://{region}.{domain}")
def azure_speech_path_manages_shared_resources(endpoint_path: str) -> bool:
return (
endpoint_path.startswith(AZURE_SPEECH_BATCH_PATH_PREFIX)
and endpoint_path != AZURE_SPEECH_FAST_TRANSCRIPTION_PATH
)
def canonical_azure_speech_endpoint_path(endpoint: str) -> str:
"""
The path Azure will actually serve, with ``.`` and ``..`` segments resolved, so the
endpoint family and the admin guard are decided on the same path the upstream request uses.
"""
raw_path: Final = httpx.URL(endpoint).path
resolved_path: Final = posixpath.normpath(f"/{raw_path.lstrip('/')}")
if raw_path.endswith("/") and resolved_path != "/":
return f"{resolved_path}/"
return resolved_path
@router.api_route(
f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list
tags=["Azure AI Speech Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list
)
async def azure_speech_proxy_route(
endpoint: str,
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
`POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
The body is forwarded byte for byte and the proxy injects its own
`Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
and is never forwarded.
[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
"""
normalized_endpoint_path: Final = canonical_azure_speech_endpoint_path(endpoint)
base_url: Final = resolve_azure_speech_base_url(
endpoint_path=normalized_endpoint_path,
api_base=get_secret_str(secret_name="AZURE_SPEECH_API_BASE"),
region=get_secret_str(secret_name="AZURE_SPEECH_REGION"),
)
if base_url is None:
raise HTTPException(
status_code=400,
detail=(
f"Unsupported Azure Speech path: {normalized_endpoint_path}. Supported prefixes are "
f"{AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX} and {AZURE_SPEECH_BATCH_PATH_PREFIX}; set "
"AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE in the proxy environment."
),
)
if azure_speech_path_manages_shared_resources(normalized_endpoint_path) and not is_proxy_admin(user_api_key_dict):
raise HTTPException(
status_code=403,
detail=(
f"{request.method} {normalized_endpoint_path} manages batch transcription resources that belong to "
"the proxy's Azure Speech subscription and whose cost is unknown at request time, so it is limited "
f"to proxy admin keys. Use {AZURE_SPEECH_FAST_TRANSCRIPTION_PATH} for transcription that is priced "
"per request."
),
)
azure_speech_api_key: Final = passthrough_endpoint_router.get_credentials(
custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
region_name=None,
)
if azure_speech_api_key is None:
raise HTTPException(
status_code=400,
detail="Azure Speech credentials not found. Set AZURE_SPEECH_API_KEY in the proxy environment.",
)
target_url: Final = base_url.copy_with(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint_path)
)
request_headers: Final = _safe_get_request_headers(request)
upstream_headers: Final = MappingProxyType(
{
header_name: header_value
for header_name, header_value in (
*(
(header_name, request_headers[header_name])
for header_name in AZURE_SPEECH_FORWARDED_REQUEST_HEADERS
if header_name in request_headers
),
(AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, azure_speech_api_key),
)
}
)
raw_body: Final = await request.body()
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=str(target_url),
custom_headers=upstream_headers,
custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
)
setattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, raw_body)
return await endpoint_func(request, fastapi_response, user_api_key_dict)
@router.post(
"/transcribe/{operation}",
tags=["Amazon Transcribe Pass-through", "pass-through"], # mutable-ok: fastapi route tags must be a list

170
litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py Normal file
View file

@ -0,0 +1,170 @@
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Final
from urllib.parse import urlparse
import httpx
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
AZURE_SPEECH_BATCH_MODEL,
AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL,
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH,
AZURE_SPEECH_MILLISECONDS_PER_SECOND,
AZURE_SPEECH_PRICING_MODEL,
AZURE_SPEECH_SHORT_AUDIO_MODEL,
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX,
AZURE_SPEECH_TICKS_PER_SECOND,
)
from litellm.cost_calculator import transcription_cost
from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload,
)
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.types.utils import StandardPassThroughResponseObject
class AzureSpeechPassthroughLoggingHandler:
@staticmethod
def _is_short_audio_route(url_route: str) -> bool:
return urlparse(url_route).path.startswith(AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX)
@staticmethod
def _is_fast_transcription_route(url_route: str) -> bool:
return urlparse(url_route).path.endswith(AZURE_SPEECH_FAST_TRANSCRIPTION_PATH)
@staticmethod
def _model_from_url_route(url_route: str) -> str:
if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route):
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_SHORT_AUDIO_MODEL}"
if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route):
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL}"
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}"
@staticmethod
def _recognized_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float:
if not isinstance(response_body, Mapping):
return 0.0
offset: Final = response_body.get("Offset")
duration: Final = response_body.get("Duration")
if not isinstance(offset, int) or not isinstance(duration, int):
return 0.0
return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND
@staticmethod
def _uploaded_audio_seconds(httpx_response: httpx.Response) -> float:
try:
uploaded_audio: Final = httpx_response.request.content
except RuntimeError:
return 0.0
return calculate_request_duration(uploaded_audio) or 0.0
@staticmethod
def _short_audio_seconds(
httpx_response: httpx.Response, response_body: Mapping[str, object] | Sequence[object] | None
) -> float:
return max(
AzureSpeechPassthroughLoggingHandler._uploaded_audio_seconds(httpx_response),
AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body),
)
@staticmethod
def _fast_transcription_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float:
if not isinstance(response_body, Mapping):
return 0.0
duration_milliseconds: Final = response_body.get("durationMilliseconds")
if not isinstance(duration_milliseconds, int):
return 0.0
return duration_milliseconds / AZURE_SPEECH_MILLISECONDS_PER_SECOND
@staticmethod
def _billed_audio_seconds(
url_route: str,
httpx_response: httpx.Response,
response_body: Mapping[str, object] | Sequence[object] | None,
) -> float:
if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route):
return AzureSpeechPassthroughLoggingHandler._short_audio_seconds(httpx_response, response_body)
if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route):
return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body)
return 0.0
@staticmethod
def _response_cost(
url_route: str,
httpx_response: httpx.Response,
response_body: Mapping[str, object] | Sequence[object] | None,
) -> float:
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(
url_route, httpx_response, response_body
)
if audio_seconds <= 0.0:
return 0.0
try:
prompt_cost, completion_cost = transcription_cost(
model=AZURE_SPEECH_PRICING_MODEL,
custom_llm_provider="azure",
duration=audio_seconds,
)
except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row
verbose_proxy_logger.warning(
"No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e
)
return 0.0
return prompt_cost + completion_cost
@staticmethod
def azure_speech_passthrough_handler(
httpx_response: httpx.Response,
response_body: Mapping[str, object] | Sequence[object] | None,
logging_obj: LiteLLMLoggingObj,
url_route: str,
result: str,
start_time: datetime,
end_time: datetime,
cache_hit: bool,
request_body: Mapping[str, object],
**kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler
) -> PassThroughEndpointLoggingTypedDict:
try:
model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route)
response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(
url_route, httpx_response, response_body
)
updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict
**kwargs,
"model": model_name,
"custom_llm_provider": AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
"response_cost": response_cost,
}
logging_obj.model_call_details.update(
model=model_name,
custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
response_cost=response_cost,
)
standard_logging_object: Final = get_standard_logging_object_payload(
kwargs=updated_kwargs,
init_response_obj=StandardPassThroughResponseObject(response=result),
start_time=start_time,
end_time=end_time,
logging_obj=logging_obj,
status="success",
)
handler_payload: Final[PassThroughEndpointLoggingTypedDict] = {
"result": StandardPassThroughResponseObject(response=result),
"kwargs": {**updated_kwargs, "standard_logging_object": standard_logging_object},
}
except Exception as e: # noqa: BLE001 # logging must never fail the forwarded request
verbose_proxy_logger.exception("Error in Azure Speech passthrough logging handler: %s", e)
fallback_payload: Final[PassThroughEndpointLoggingTypedDict] = {
"result": StandardPassThroughResponseObject(response=result),
"kwargs": kwargs,
}
return fallback_payload
return handler_payload

3
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -53,6 +53,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_no_callback_env_reference
from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import _get_masked_values
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.base_llm.managed_resources.utils import (
@ -1024,7 +1025,7 @@ async def pass_through_request(
verbose_proxy_logger.debug(
"Pass through endpoint sending request to \nURL %s\nheaders: %s\nbody: %s\n",
url,
upstream_headers,
_get_masked_values(upstream_headers),
_parsed_body,
)

25
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -6,6 +6,7 @@ from urllib.parse import urlparse
import httpx
from litellm.constants import AZURE_SPEECH_CUSTOM_LLM_PROVIDER
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import PassThroughEndpointLoggingResultValues
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@ -274,6 +275,25 @@ class PassThroughEndpointLogging:
)
standard_logging_response_object = comprehend_medical_handler_result["result"] # rebind-ok: elif-chain
kwargs = comprehend_medical_handler_result["kwargs"] # rebind-ok: elif-chain contract
elif self.is_azure_speech_route(custom_llm_provider):
from .llm_provider_handlers.azure_speech_passthrough_logging_handler import (
AzureSpeechPassthroughLoggingHandler,
)
azure_speech_handler_result: Final = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=httpx_response,
response_body=response_body,
logging_obj=logging_obj,
url_route=url_route,
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
request_body=request_body,
**kwargs,
)
standard_logging_response_object = azure_speech_handler_result["result"] # rebind-ok: elif-chain
kwargs = azure_speech_handler_result["kwargs"] # rebind-ok: elif-chain contract
elif self.is_transcribe_route(custom_llm_provider):
transcribe_handler_result: Final = TranscribePassthroughLoggingHandler.transcribe_passthrough_handler(
httpx_response=httpx_response,
@ -351,7 +371,7 @@ class PassThroughEndpointLogging:
):
standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None
logging_obj.model_call_details["passthrough_logging_payload"] = passthrough_logging_payload
if self.is_assemblyai_route(url_route):
if self.is_assemblyai_route(url_route) and not self.is_azure_speech_route(custom_llm_provider):
if AssemblyAIPassthroughLoggingHandler._should_log_request(httpx_response.request.method) is not True:
return
self.assemblyai_passthrough_logging_handler.assemblyai_passthrough_logging_handler(
@ -458,6 +478,9 @@ class PassThroughEndpointLogging:
def is_comprehend_medical_route(self, custom_llm_provider: str | None) -> bool:
return custom_llm_provider == "comprehendmedical"
def is_azure_speech_route(self, custom_llm_provider: str | None) -> bool:
return custom_llm_provider == AZURE_SPEECH_CUSTOM_LLM_PROVIDER
def is_transcribe_route(self, custom_llm_provider: str | None) -> bool:
return custom_llm_provider == TRANSCRIBE_CUSTOM_LLM_PROVIDER

18
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -586,6 +586,24 @@
],
"default_model_placeholder": "azure_ai/command-r-plus"
},
{
"provider": "Azure_Speech",
"provider_display_name": "Azure AI Speech",
"litellm_provider": "azure_speech",
"credential_fields": [
{
"key": "api_key",
"label": "Azure AI Speech Subscription Key",
"placeholder": null,
"tooltip": "The Ocp-Apim-Subscription-Key for your Azure AI Speech resource. The proxy injects it on every /azure_speech/* pass-through request. Region and API base come from AZURE_SPEECH_REGION / AZURE_SPEECH_API_BASE",
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "azure_speech/short-audio"
},
{
"provider": "AZURE_TEXT",
"provider_display_name": "Azure Text",

1
litellm/types/utils.py
View file

@ -4074,6 +4074,7 @@ class LlmProviders(str, Enum):
TOPAZ = "topaz"
SAP_GENERATIVE_AI_HUB = "sap"
ASSEMBLYAI = "assemblyai"
AZURE_SPEECH = "azure_speech"
CHARITY_ENGINE = "charity_engine"
GITHUB_COPILOT = "github_copilot"
SNOWFLAKE = "snowflake"

2
terraform/litellm/aws/locals.tf
View file

@ -86,7 +86,7 @@ locals {
"/queue/chat/*",
"/v1beta/*",
"/interactions/*",
"/anthropic/*", "/azure/*", "/azure_ai/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", "/transcribe*",
"/anthropic/*", "/azure/*", "/azure_ai/*", "/azure_speech/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", "/transcribe*",
"/cohere/*", "/gemini/*", "/google/*",
"/vertex_ai/*", "/vertex-ai/*",
"/assemblyai/*", "/eu.assemblyai/*",

2
terraform/litellm/gcp/locals.tf
View file

@ -55,7 +55,7 @@ locals {
"/queue/chat/*",
"/v1beta/*",
"/interactions/*",
"/anthropic/*", "/azure/*", "/azure_ai/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", "/transcribe*",
"/anthropic/*", "/azure/*", "/azure_ai/*", "/azure_speech/*", "/aws/*", "/bedrock/*", "/comprehendmedical*", "/transcribe*",
"/cohere/*", "/gemini/*", "/google/*",
"/vertex_ai/*", "/vertex-ai/*",
"/assemblyai/*", "/eu.assemblyai/*",

5
tests/test_litellm/proxy/middleware/test_billable_request_metrics_middleware.py
View file

@ -116,6 +116,11 @@ def test_is_pure_asgi_not_base_http_middleware():
# Bare AWS-SDK-shaped route carries the operation in X-Amz-Target and writes SpendLogs
("/comprehendmedical", (BillableCategory.LLM, "/comprehendmedical")),
("/comprehendmedical/DetectEntitiesV2", (BillableCategory.LLM, "/comprehendmedical")),
(
"/azure_speech/speech/recognition/conversation/cognitiveservices/v1",
(BillableCategory.LLM, "/azure_speech"),
),
("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")),
("/transcribe", (BillableCategory.LLM, "/transcribe")),
("/transcribe/StartTranscriptionJob", (BillableCategory.LLM, "/transcribe")),
("/mcp", (BillableCategory.MCP, "/mcp")),

292
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py Normal file
View file

@ -0,0 +1,292 @@
import io
import json
import wave
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.azure_speech_passthrough_logging_handler import (
AzureSpeechPassthroughLoggingHandler,
)
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
SHORT_AUDIO_URL = (
"https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US"
)
BATCH_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2/transcriptions"
FAST_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
FAST_BODY = {"durationMilliseconds": 5061, "combinedPhrases": [{"text": "Hello world."}]}
FAST_AUDIO_SECONDS = 5.061
TRANSCRIPT_BODY = {
"RecognitionStatus": "Success",
"Offset": 5000000,
"Duration": 25000000,
"DisplayText": "Hello world.",
}
TRANSCRIPT = json.dumps(TRANSCRIPT_BODY)
TRANSCRIPT_AUDIO_SECONDS = 3.0
PRICE_PER_SECOND = 0.5
WAV_SAMPLE_RATE: Final = 16000
UNRECOGNIZED_BODIES: Final = (
{"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0},
{"RecognitionStatus": "InitialSilenceTimeout"},
{"Offset": "5000000", "Duration": "25000000"},
{},
[],
None,
)
def _pcm16_wav(seconds: float) -> bytes:
buffer: Final = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(WAV_SAMPLE_RATE)
wav.writeframes(b"\x00\x00" * int(seconds * WAV_SAMPLE_RATE))
return buffer.getvalue()
@pytest.fixture(autouse=True)
def azure_stt_price(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setitem(
litellm.model_cost,
"azure/speech/azure-stt",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": PRICE_PER_SECOND,
"output_cost_per_second": 0.0,
},
)
def _make_response(url: str, uploaded: bytes = b"") -> httpx.Response:
request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}, content=uploaded)
return httpx.Response(200, request=request, text=TRANSCRIPT)
def _make_logging_obj() -> MagicMock:
logging_obj = MagicMock()
logging_obj.litellm_call_id = "test-call-id"
logging_obj.model_call_details = {}
return logging_obj
class TestAzureSpeechPassthroughHandler:
@pytest.mark.parametrize(
"url_route,expected_model,expected_cost",
[
(SHORT_AUDIO_URL, "azure_speech/short-audio", TRANSCRIPT_AUDIO_SECONDS * PRICE_PER_SECOND),
(FAST_URL, "azure_speech/fast-transcription", FAST_AUDIO_SECONDS * PRICE_PER_SECOND),
(BATCH_URL, "azure_speech/batch-transcription", 0.0),
(f"{BATCH_URL}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files", "azure_speech/batch-transcription", 0.0),
],
)
def test_records_model_provider_and_cost(self, url_route: str, expected_model: str, expected_cost: float):
logging_obj = _make_logging_obj()
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(url_route),
response_body={**TRANSCRIPT_BODY, **FAST_BODY},
logging_obj=logging_obj,
url_route=url_route,
result=TRANSCRIPT,
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["result"] == {"response": TRANSCRIPT}
assert handler_result["kwargs"]["model"] == expected_model
assert handler_result["kwargs"]["custom_llm_provider"] == "azure_speech"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_cost)
assert handler_result["kwargs"]["standard_logging_object"]["response_cost"] == pytest.approx(expected_cost)
assert handler_result["kwargs"]["standard_logging_object"]["model"] == expected_model
assert logging_obj.model_call_details["model"] == expected_model
assert logging_obj.model_call_details["custom_llm_provider"] == "azure_speech"
assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost)
@pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES)
@pytest.mark.parametrize("uploaded", [b"", b"not audio at all"])
def test_short_audio_with_neither_recognized_nor_decodable_audio_logs_zero_cost(
self, response_body: dict[str, object] | list[dict[str, object]] | None, uploaded: bytes
):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(SHORT_AUDIO_URL, uploaded),
response_body=response_body,
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result="",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["model"] == "azure_speech/short-audio"
assert handler_result["kwargs"]["response_cost"] == 0.0
@pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES)
def test_short_audio_bills_the_uploaded_audio_when_nothing_was_recognized(
self, response_body: dict[str, object] | list[dict[str, object]] | None
):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=2.0)),
response_body=response_body,
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result="",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["response_cost"] == pytest.approx(2.0 * PRICE_PER_SECOND)
@pytest.mark.parametrize(
"uploaded_seconds,expected_seconds",
[(1.0, TRANSCRIPT_AUDIO_SECONDS), (TRANSCRIPT_AUDIO_SECONDS + 2.0, TRANSCRIPT_AUDIO_SECONDS + 2.0)],
)
def test_short_audio_bills_the_longer_of_uploaded_and_recognized_audio(
self, uploaded_seconds: float, expected_seconds: float
):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=uploaded_seconds)),
response_body=TRANSCRIPT_BODY,
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result=TRANSCRIPT,
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_seconds * PRICE_PER_SECOND)
def test_fast_transcription_ignores_the_uploaded_multipart_body(self):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(FAST_URL, _pcm16_wav(seconds=30.0)),
response_body=FAST_BODY,
logging_obj=_make_logging_obj(),
url_route=FAST_URL,
result=json.dumps(FAST_BODY),
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND)
@pytest.mark.parametrize(
"response_body",
[{"durationMilliseconds": 0}, {"durationMilliseconds": "5061"}, {"duration": 5061}, {}, [], None],
)
def test_fast_transcription_without_duration_milliseconds_logs_zero_cost(
self, response_body: dict[str, object] | list[dict[str, object]] | None
):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(FAST_URL),
response_body=response_body,
logging_obj=_make_logging_obj(),
url_route=FAST_URL,
result="",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription"
assert handler_result["kwargs"]["response_cost"] == 0.0
def test_missing_price_entry_still_logs_the_row_at_zero_cost(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.delitem(litellm.model_cost, "azure/speech/azure-stt")
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(SHORT_AUDIO_URL),
response_body=TRANSCRIPT_BODY,
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result=TRANSCRIPT,
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["model"] == "azure_speech/short-audio"
assert handler_result["kwargs"]["custom_llm_provider"] == "azure_speech"
assert handler_result["kwargs"]["response_cost"] == 0.0
def test_subscription_key_never_reaches_the_logging_payload(self):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(SHORT_AUDIO_URL),
response_body=TRANSCRIPT_BODY,
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result=TRANSCRIPT,
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert "server-secret" not in repr(handler_result)
class TestIsAzureSpeechRoute:
def test_matches_by_provider_tag(self):
assert PassThroughEndpointLogging().is_azure_speech_route("azure_speech")
@pytest.mark.parametrize("provider", ["azure", "azure_ai", "comprehendmedical", None])
def test_does_not_match_other_providers(self, provider: str | None):
assert not PassThroughEndpointLogging().is_azure_speech_route(provider)
def test_config_driven_passthrough_to_azure_speech_host_is_not_claimed(self):
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
httpx_response=_make_response(SHORT_AUDIO_URL),
response_body={"RecognitionStatus": "Success"},
request_body={},
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result=TRANSCRIPT,
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
custom_llm_provider=None,
)
assert normalized["kwargs"].get("model") != "azure_speech/short-audio"
assert "response_cost" not in normalized["kwargs"]
class TestNormalizeDispatch:
def test_normalize_routes_to_azure_speech_handler(self):
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
httpx_response=_make_response(SHORT_AUDIO_URL),
response_body=TRANSCRIPT_BODY,
request_body={},
logging_obj=_make_logging_obj(),
url_route=SHORT_AUDIO_URL,
result="",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
custom_llm_provider="azure_speech",
)
assert normalized["standard_logging_response_object"] == {"response": ""}
assert normalized["kwargs"]["model"] == "azure_speech/short-audio"
assert normalized["kwargs"]["custom_llm_provider"] == "azure_speech"
assert normalized["kwargs"]["response_cost"] == pytest.approx(TRANSCRIPT_AUDIO_SECONDS * PRICE_PER_SECOND)

597
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -2,6 +2,7 @@ import asyncio
import base64
import contextlib
import json
import logging
import os
import traceback
from collections.abc import Iterator, Mapping
@ -32,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
_proxy_general_settings,
anthropic_proxy_route,
azure_proxy_route,
azure_speech_proxy_route,
bedrock_llm_proxy_route,
bedrock_proxy_route,
create_pass_through_route,
@ -6435,6 +6437,601 @@ class TestAzureRelayDeploymentSegment:
assert [call["model"] for call in captured] == ["gpt", "gpt"]
AZURE_SPEECH_SHORT_AUDIO_ENDPOINT: Final = "/speech/recognition/conversation/cognitiveservices/v1"
AZURE_SPEECH_BATCH_ENDPOINT: Final = "/speechtotext/v3.2/transcriptions"
AZURE_SPEECH_FAST_ENDPOINT: Final = "/speechtotext/transcriptions:transcribe"
AZURE_SPEECH_PCM16_HEADER: Final = (
b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00"
)
AZURE_SPEECH_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + b"\x00" * 3072
AZURE_SPEECH_WAV_SECONDS: Final = 3072 / (16000 * 2)
AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range(256)) * 12
AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."}
def _azure_speech_test_client(monkeypatch: pytest.MonkeyPatch, caller: UserAPIKeyAuth) -> TestClient:
from litellm.proxy.proxy_server import app
monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key")
monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus")
monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False)
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: caller)
return TestClient(app)
@pytest.fixture
def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
yield _azure_speech_test_client(monkeypatch, UserAPIKeyAuth(api_key="sk-virtual"))
@pytest.fixture
def azure_speech_admin_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
yield _azure_speech_test_client(
monkeypatch, UserAPIKeyAuth(api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN)
)
class TestAzureSpeechProxyRoute:
"""Drives the real FastAPI route with respx standing in for the Azure hosts only."""
def test_short_audio_forwards_raw_wav_bytes_with_server_key(self, azure_speech_client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(
f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}"
).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT))
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
params={"language": "en-US", "format": "detailed"},
content=AZURE_SPEECH_WAV_BYTES,
headers={
"Content-Type": "audio/wav; codecs=audio/pcm; samplerate=16000",
"Authorization": "Bearer sk-virtual",
"Ocp-Apim-Subscription-Key": "caller-supplied-key",
"x-pass-ocp-apim-subscription-key": "caller-supplied-key",
},
)
assert (response.status_code, response.json()) == (200, AZURE_SPEECH_TRANSCRIPT)
sent = route.calls.last.request
assert sent.content == AZURE_SPEECH_WAV_BYTES
assert dict(sent.url.params) == {"language": "en-US", "format": "detailed"}
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
assert sent.headers["content-type"] == "audio/wav; codecs=audio/pcm; samplerate=16000"
assert "authorization" not in sent.headers
assert "caller-supplied-key" not in repr(sent.headers)
def test_admin_batch_job_creation_goes_to_the_cognitive_services_host(
self, azure_speech_admin_client: TestClient
) -> None:
body: Final = {"contentUrls": ["https://example.com/a.wav"], "locale": "en-US", "displayName": "job"}
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock(
return_value=httpx.Response(201, json={"self": "https://eastus.api.cognitive.microsoft.com/x"})
)
response = azure_speech_admin_client.post(
f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}",
json=body,
headers={"Authorization": "Bearer sk-admin"},
)
assert response.status_code == 201
sent = route.calls.last.request
assert json.loads(sent.content) == body
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
assert "authorization" not in sent.headers
@pytest.mark.parametrize(
"method,endpoint",
[
("POST", AZURE_SPEECH_BATCH_ENDPOINT),
("POST", "/speechtotext/v3.2/models"),
("PUT", "/speechtotext/v3.2/endpoints/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"),
("GET", AZURE_SPEECH_BATCH_ENDPOINT),
("GET", f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files"),
("PATCH", f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"),
("DELETE", f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"),
],
)
def test_non_admin_key_cannot_manage_shared_batch_resources(
self, azure_speech_client: TestClient, method: str, endpoint: str
) -> None:
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(200, json={"status": "Succeeded"}))
response = azure_speech_client.request(
method,
f"/azure_speech{endpoint}",
json={"contentUrls": ["https://example.com/a.wav"], "locale": "en-US"},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 403, response.text
assert AZURE_SPEECH_FAST_ENDPOINT in response.text
assert not catch_all.called
def test_non_admin_key_can_still_fast_transcribe_in_the_batch_family(self, azure_speech_client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []})
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
def test_admin_key_reads_and_deletes_batch_jobs(self, azure_speech_admin_client: TestClient) -> None:
job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"
with respx.mock(assert_all_called=True) as upstream:
upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock(
return_value=httpx.Response(200, json={"status": "Succeeded"})
)
upstream.delete(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock(
return_value=httpx.Response(204)
)
statuses = [
azure_speech_admin_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-admin"}),
azure_speech_admin_client.delete(
f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-admin"}
),
]
assert [r.status_code for r in statuses] == [200, 204]
def test_fast_transcription_multipart_upload_is_forwarded_byte_for_byte(
self, azure_speech_client: TestClient
) -> None:
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []})
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
sent = route.calls.last.request
assert sent.headers["content-type"].startswith("multipart/form-data; boundary=")
assert dict(sent.url.params) == {"api-version": "2024-11-15"}
assert AZURE_SPEECH_NON_UTF8_WAV_BYTES in sent.content
assert b'name="definition"' in sent.content
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
assert "authorization" not in sent.headers
def test_batch_get_is_forwarded_with_the_job_id_path(self, azure_speech_admin_client: TestClient) -> None:
job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files"
with respx.mock(assert_all_called=True) as upstream:
route = upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock(
return_value=httpx.Response(200, json={"values": []})
)
response = azure_speech_admin_client.get(
f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-admin"}
)
assert (response.status_code, response.json()) == (200, {"values": []})
assert route.calls.last.request.headers["ocp-apim-subscription-key"] == "server-subscription-key"
@pytest.mark.parametrize("method", ["GET", "POST"])
def test_batch_requests_are_logged_as_azure_speech_not_assemblyai(
self, azure_speech_admin_client: TestClient, monkeypatch: pytest.MonkeyPatch, method: str
) -> None:
from litellm.integrations.custom_logger import CustomLogger
class _Recorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
self.payloads.append(kwargs["standard_logging_object"])
recorder: Final = _Recorder()
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder])
with respx.mock(assert_all_called=True) as upstream:
upstream.request(method, f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"values": []})
)
response = azure_speech_admin_client.request(
method,
f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}",
json={"locale": "en-US"} if method == "POST" else None,
headers={"Authorization": "Bearer sk-admin"},
)
assert response.status_code == 200
assert [(p["model"], p["custom_llm_provider"], p["response_cost"]) for p in recorder.payloads] == [
("azure_speech/batch-transcription", "azure_speech", 0.0)
]
def test_fast_transcription_spend_is_priced_from_duration_milliseconds(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.integrations.custom_logger import CustomLogger
class _Recorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
self.payloads.append(kwargs["standard_logging_object"])
recorder: Final = _Recorder()
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder])
monkeypatch.setitem(
litellm.model_cost,
"azure/speech/azure-stt",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": 0.25,
"output_cost_per_second": 0.0,
},
)
with respx.mock(assert_all_called=True) as upstream:
upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []})
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [
("azure_speech/fast-transcription", "azure_speech")
]
assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * 0.25)
def test_short_audio_spend_is_priced_from_the_recognized_duration(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.integrations.custom_logger import CustomLogger
class _Recorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
self.payloads.append(kwargs["standard_logging_object"])
recorder: Final = _Recorder()
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder])
monkeypatch.setitem(
litellm.model_cost,
"azure/speech/azure-stt",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": 0.25,
"output_cost_per_second": 0.0,
},
)
transcript: Final = {**AZURE_SPEECH_TRANSCRIPT, "Offset": 10_000_000, "Duration": 30_000_000}
with respx.mock(assert_all_called=True) as upstream:
upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock(
return_value=httpx.Response(200, json=transcript)
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
content=AZURE_SPEECH_WAV_BYTES,
headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [
("azure_speech/short-audio", "azure_speech")
]
assert recorder.payloads[0]["response_cost"] == pytest.approx(4.0 * 0.25)
def test_api_base_wins_over_region_for_both_families(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv("AZURE_SPEECH_API_BASE", "https://my-speech.cognitiveservices.azure.com")
with respx.mock(assert_all_called=True) as upstream:
short_audio = upstream.post(
f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}"
).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT))
fast = upstream.post(f"https://my-speech.cognitiveservices.azure.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []})
)
azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
content=AZURE_SPEECH_WAV_BYTES,
headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"},
)
azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
)
assert short_audio.called and fast.called
@pytest.mark.parametrize("endpoint", ["openai/deployments/whisper/audio/transcriptions", "speech", "speechtotext"])
def test_unknown_path_family_is_rejected_before_any_upstream_call(
self, azure_speech_client: TestClient, endpoint: str
) -> None:
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(200))
response = azure_speech_client.post(
f"/azure_speech/{endpoint}", content=b"x", headers={"Authorization": "Bearer sk-virtual"}
)
assert response.status_code == 400
assert not catch_all.called
def test_missing_region_and_base_is_rejected_before_any_upstream_call(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("AZURE_SPEECH_REGION")
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(200))
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
content=AZURE_SPEECH_WAV_BYTES,
headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 400
assert "AZURE_SPEECH_REGION" in response.text
assert not catch_all.called
def test_missing_api_key_is_rejected_before_any_upstream_call(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("AZURE_SPEECH_API_KEY")
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(200))
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
content=AZURE_SPEECH_WAV_BYTES,
headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 400
assert "AZURE_SPEECH_API_KEY" in response.text
assert not catch_all.called
def test_azure_speech_is_a_mapped_pass_through_route(self) -> None:
from litellm.proxy._types import LiteLLMRoutes
assert "/azure_speech" in LiteLLMRoutes.mapped_pass_through_routes.value
def test_short_audio_with_no_recognized_speech_is_billed_for_the_uploaded_audio(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.integrations.custom_logger import CustomLogger
class _Recorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
self.payloads.append(kwargs["standard_logging_object"])
recorder: Final = _Recorder()
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder])
monkeypatch.setitem(
litellm.model_cost,
"azure/speech/azure-stt",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": 0.25,
"output_cost_per_second": 0.0,
},
)
with respx.mock(assert_all_called=True) as upstream:
upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0})
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
content=AZURE_SPEECH_WAV_BYTES,
headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
assert [p["model"] for p in recorder.payloads] == ["azure_speech/short-audio"]
assert recorder.payloads[0]["response_cost"] == pytest.approx(AZURE_SPEECH_WAV_SECONDS * 0.25)
class TestAzureSpeechProxyRoutePathTraversal:
"""Calls the route function directly because httpx clients resolve dot segments before sending."""
@pytest.mark.parametrize(
"endpoint",
[
f"speech/..{AZURE_SPEECH_BATCH_ENDPOINT}",
f"speech/recognition/../..{AZURE_SPEECH_BATCH_ENDPOINT}/",
f"speech/./..{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab",
],
)
@pytest.mark.asyncio
async def test_dot_segments_cannot_reach_shared_batch_resources_with_a_non_admin_key(
self, monkeypatch: pytest.MonkeyPatch, endpoint: str
) -> None:
monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key")
monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus")
monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False)
request: Final = MagicMock(spec=Request)
request.method = "GET"
with pytest.raises(HTTPException) as denied:
await azure_speech_proxy_route(
endpoint=endpoint,
request=request,
fastapi_response=MagicMock(spec=Response),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-virtual"),
)
assert denied.value.status_code == 403
assert AZURE_SPEECH_FAST_ENDPOINT in str(denied.value.detail)
def _azure_speech_real_auth_attrs() -> dict[str, object]:
from litellm.caching.caching import DualCache
from litellm.proxy.utils import ProxyLogging
user_api_key_cache: Final = DualCache()
return {
"prisma_client": None,
"user_api_key_cache": user_api_key_cache,
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
"master_key": "sk-master-key",
"general_settings": {},
"llm_model_list": [],
"llm_router": None,
"open_telemetry_logger": None,
"user_custom_auth": None,
"jwt_handler": None,
}
class TestAzureSpeechRawBodyThroughRealAuth:
"""user_api_key_auth reads the body before the route runs; raw audio must not be parsed as JSON."""
def _post(
self, monkeypatch: pytest.MonkeyPatch, path: str, api_key: str, content_type: str, body: bytes
) -> httpx.Response:
from litellm.proxy.proxy_server import app
monkeypatch.delitem(app.dependency_overrides, user_api_key_auth, raising=False)
monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key")
monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus")
monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False)
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
with patch.multiple( # test-quality-ok: the real user_api_key_auth reads proxy_server module globals (master_key, caches) that have no injection seam
"litellm.proxy.proxy_server", **_azure_speech_real_auth_attrs()
):
client = TestClient(app)
return client.post(
path,
params={"language": "en-US"},
content=body,
headers={"Content-Type": content_type, "Authorization": f"Bearer {api_key}"},
)
def _post_wav(
self, monkeypatch: pytest.MonkeyPatch, path: str, api_key: str, body: bytes = AZURE_SPEECH_WAV_BYTES
) -> httpx.Response:
return self._post(monkeypatch, path, api_key, "audio/wav", body)
@pytest.mark.parametrize("body", [AZURE_SPEECH_WAV_BYTES, AZURE_SPEECH_NON_UTF8_WAV_BYTES], ids=["ascii", "binary"])
def test_master_key_with_raw_wav_body_reaches_azure_without_a_parse_attempt(
self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, body: bytes
) -> None:
with respx.mock(assert_all_called=True) as upstream, caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"):
route = upstream.post(
f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}"
).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT))
response = self._post_wav(
monkeypatch, f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", "sk-master-key", body=body
)
assert (response.status_code, response.json()) == (200, AZURE_SPEECH_TRANSCRIPT)
assert route.calls.last.request.content == body
assert [record.message for record in caplog.records if "request body" in record.message] == []
def test_wrong_litellm_key_with_raw_wav_body_is_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(200))
response = self._post_wav(monkeypatch, f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", "sk-wrong")
assert response.status_code in (400, 401), response.text
assert not catch_all.called
def test_master_key_with_multipart_batch_upload_is_forwarded_byte_for_byte(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
boundary: Final = "lit7939boundary"
multipart_body: Final = (
f"--{boundary}\r\nContent-Disposition: form-data; name=\"definition\"\r\n\r\n".encode()
+ json.dumps({"locales": ["en-US"]}).encode()
+ f"\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"audio\"; filename=\"eagle.wav\"\r\n"
"Content-Type: audio/wav\r\n\r\n".encode()
+ AZURE_SPEECH_NON_UTF8_WAV_BYTES
+ f"\r\n--{boundary}--\r\n".encode()
)
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock(
return_value=httpx.Response(201, json={"status": "NotStarted"})
)
response = self._post(
monkeypatch,
f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}",
"sk-master-key",
f"multipart/form-data; boundary={boundary}",
multipart_body,
)
assert (response.status_code, response.json()) == (201, {"status": "NotStarted"})
sent = route.calls.last.request
assert sent.content == multipart_body
assert sent.headers["content-type"] == f"multipart/form-data; boundary={boundary}"
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
@pytest.mark.parametrize("content_type", ["audio/wav", "multipart/form-data; boundary=x"])
def test_wrong_litellm_key_with_multipart_batch_upload_is_rejected(
self, monkeypatch: pytest.MonkeyPatch, content_type: str
) -> None:
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(200))
response = self._post(
monkeypatch, f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}", "sk-wrong", content_type, b"--x--\r\n"
)
assert response.status_code in (400, 401), response.text
assert not catch_all.called
def test_audio_content_type_off_the_azure_speech_route_is_still_parsed_as_json(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
response = self._post_wav(monkeypatch, "/v1/chat/completions", "sk-master-key", body=b'{}{"model": "gpt-4o"}')
assert response.status_code == 400
assert "Invalid JSON payload" in response.text
class TestTypeSafePassthroughRoute:
@staticmethod
def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock:

16
tests/test_litellm/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py
View file

@ -159,6 +159,22 @@ def test_assemblyai_region_matching():
assert passthrough_router.get_credentials(custom_llm_provider="assemblyai", region_name=None) == "sk-us"
def test_azure_speech_dashboard_credential_resolves_through_flagged_deployment(monkeypatch):
monkeypatch.delenv("AZURE_SPEECH_API_KEY", raising=False)
CredentialAccessor.upsert_credentials([_credential("azure-speech-prod", "azure-subscription-key")])
llm_router = litellm.Router(
model_list=[
_flagged_deployment("azure_speech/short-audio", litellm_credential_name="azure-speech-prod"),
]
)
passthrough_router = _passthrough_router(llm_router)
assert (
passthrough_router.get_credentials(custom_llm_provider="azure_speech", region_name=None)
== "azure-subscription-key"
)
def test_env_fallback_when_no_router(monkeypatch):
passthrough_router = _passthrough_router(None)
monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")

4
ui/litellm-dashboard/src/components/provider_info_helpers.tsx
View file

@ -80,6 +80,7 @@ export enum Providers {
SageMaker = "AWS SageMaker",
Azure = "Azure",
Azure_AI_Studio = "Azure AI Foundry (Studio)",
Azure_Speech = "Azure AI Speech",
AZURE_TEXT = "Azure Text",
BASETEN = "Baseten",
BYTEZ = "Bytez",
@ -193,6 +194,7 @@ export const provider_map: Record<string, string> = {
AUTO_ROUTER: "auto_router",
Azure: "azure",
Azure_AI_Studio: "azure_ai",
Azure_Speech: "azure_speech",
AZURE_TEXT: "azure_text",
BASETEN: "baseten",
Bedrock: "bedrock",
@ -310,6 +312,7 @@ export const providerLogoMap: Partial<Record<Providers, string>> = {
[Providers.AssemblyAI]: assemblyaiSmallLogo.src,
[Providers.Azure]: microsoftAzureLogo.src,
[Providers.Azure_AI_Studio]: microsoftAzureLogo.src,
[Providers.Azure_Speech]: microsoftAzureLogo.src,
[Providers.AZURE_TEXT]: microsoftAzureLogo.src,
[Providers.BASETEN]: basetenLogo.src,
[Providers.Bedrock]: bedrockLogo.src,
@ -427,6 +430,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
[Providers.Anthropic]: "claude-3-opus",
[Providers.Azure]: "my-deployment",
[Providers.Azure_AI_Studio]: "azure_ai/command-r-plus",
[Providers.Azure_Speech]: "azure_speech/short-audio",
[Providers.Bedrock]: "claude-3-opus",
[Providers.CHATGPT]: "chatgpt/gpt-5.4",
[Providers.Cognition]: "cognition/swe-1.7",

231
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -1612,6 +1612,82 @@ export interface paths {
patch: operations["azure_proxy_route_azure_ai__endpoint__patch"];
trace?: never;
};
"/azure_speech/{endpoint}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Azure Speech Proxy Route
* @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
* `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
* with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
*
* The body is forwarded byte for byte and the proxy injects its own
* `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
* and is never forwarded.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
*/
get: operations["azure_speech_proxy_route_azure_speech__endpoint__get"];
/**
* Azure Speech Proxy Route
* @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
* `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
* with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
*
* The body is forwarded byte for byte and the proxy injects its own
* `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
* and is never forwarded.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
*/
put: operations["azure_speech_proxy_route_azure_speech__endpoint__put"];
/**
* Azure Speech Proxy Route
* @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
* `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
* with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
*
* The body is forwarded byte for byte and the proxy injects its own
* `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
* and is never forwarded.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
*/
post: operations["azure_speech_proxy_route_azure_speech__endpoint__post"];
/**
* Azure Speech Proxy Route
* @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
* `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
* with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
*
* The body is forwarded byte for byte and the proxy injects its own
* `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
* and is never forwarded.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
*/
delete: operations["azure_speech_proxy_route_azure_speech__endpoint__delete"];
options?: never;
head?: never;
/**
* Azure Speech Proxy Route
* @description Pass-through for the Azure AI Speech REST APIs (speech to text), e.g.
* `POST /azure_speech/speech/recognition/conversation/cognitiveservices/v1?language=en-US`
* with the raw audio as the body, or `POST /azure_speech/speechtotext/v3.2/transcriptions`.
*
* The body is forwarded byte for byte and the proxy injects its own
* `Ocp-Apim-Subscription-Key`; the caller's `Authorization` header is the LiteLLM key
* and is never forwarded.
*
* [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
*/
patch: operations["azure_speech_proxy_route_azure_speech__endpoint__patch"];
trace?: never;
};
"/batches": {
parameters: {
query?: never;
@ -43889,6 +43965,161 @@ export interface operations {
};
};
};
azure_speech_proxy_route_azure_speech__endpoint__get: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
azure_speech_proxy_route_azure_speech__endpoint__put: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
azure_speech_proxy_route_azure_speech__endpoint__post: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
azure_speech_proxy_route_azure_speech__endpoint__delete: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
azure_speech_proxy_route_azure_speech__endpoint__patch: {
parameters: {
query?: never;
header?: never;
path: {
endpoint: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": unknown;
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
list_batches_batches_get: {
parameters: {
query?: {