diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index ab4b636cb60..ed70369506d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -12,6 +12,7 @@ import hmac import inspect import json import os +import posixpath import re from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass @@ -1408,6 +1409,18 @@ def azure_speech_path_manages_shared_resources(endpoint_path: str) -> bool: ) +def canonical_azure_speech_endpoint_path(endpoint: str) -> str: + """ + The path Azure will actually serve, with ``.`` and ``..`` segments resolved, so the + endpoint family and the admin guard are decided on the same path the upstream request uses. + """ + raw_path: Final = httpx.URL(endpoint).path + resolved_path: Final = posixpath.normpath(f"/{raw_path.lstrip('/')}") + if raw_path.endswith("/") and resolved_path != "/": + return f"{resolved_path}/" + return resolved_path + + @router.api_route( f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list @@ -1430,8 +1443,7 @@ async def azure_speech_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/azure_speech) """ - endpoint_path: Final = httpx.URL(endpoint).path - normalized_endpoint_path: Final = endpoint_path if endpoint_path.startswith("/") else f"/{endpoint_path}" + normalized_endpoint_path: Final = canonical_azure_speech_endpoint_path(endpoint) base_url: Final = resolve_azure_speech_base_url( endpoint_path=normalized_endpoint_path, api_base=get_secret_str(secret_name="AZURE_SPEECH_API_BASE"), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py index 588b7cc8e56..8cd1b137de8 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -18,6 +18,7 @@ from litellm.constants import ( AZURE_SPEECH_TICKS_PER_SECOND, ) from litellm.cost_calculator import transcription_cost +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, @@ -53,6 +54,23 @@ class AzureSpeechPassthroughLoggingHandler: return 0.0 return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND + @staticmethod + def _uploaded_audio_seconds(httpx_response: httpx.Response) -> float: + try: + uploaded_audio: Final = httpx_response.request.content + except RuntimeError: + return 0.0 + return calculate_request_duration(uploaded_audio) or 0.0 + + @staticmethod + def _short_audio_seconds( + httpx_response: httpx.Response, response_body: Mapping[str, object] | Sequence[object] | None + ) -> float: + return max( + AzureSpeechPassthroughLoggingHandler._uploaded_audio_seconds(httpx_response), + AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body), + ) + @staticmethod def _fast_transcription_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float: if not isinstance(response_body, Mapping): @@ -63,16 +81,26 @@ class AzureSpeechPassthroughLoggingHandler: return duration_milliseconds / AZURE_SPEECH_MILLISECONDS_PER_SECOND @staticmethod - def _billed_audio_seconds(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: + def _billed_audio_seconds( + url_route: str, + httpx_response: httpx.Response, + response_body: Mapping[str, object] | Sequence[object] | None, + ) -> float: if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route): - return AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body) + return AzureSpeechPassthroughLoggingHandler._short_audio_seconds(httpx_response, response_body) if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route): return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body) return 0.0 @staticmethod - def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float: - audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(url_route, response_body) + def _response_cost( + url_route: str, + httpx_response: httpx.Response, + response_body: Mapping[str, object] | Sequence[object] | None, + ) -> float: + audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds( + url_route, httpx_response, response_body + ) if audio_seconds <= 0.0: return 0.0 try: @@ -103,7 +131,9 @@ class AzureSpeechPassthroughLoggingHandler: ) -> PassThroughEndpointLoggingTypedDict: try: model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) - response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(url_route, response_body) + response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost( + url_route, httpx_response, response_body + ) updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict **kwargs, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py index 50d3de64f72..670f8e65823 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -1,5 +1,8 @@ +import io import json +import wave from datetime import datetime +from typing import Final from unittest.mock import MagicMock import httpx @@ -29,6 +32,25 @@ TRANSCRIPT_BODY = { TRANSCRIPT = json.dumps(TRANSCRIPT_BODY) TRANSCRIPT_AUDIO_SECONDS = 3.0 PRICE_PER_SECOND = 0.5 +WAV_SAMPLE_RATE: Final = 16000 +UNRECOGNIZED_BODIES: Final = ( + {"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}, + {"RecognitionStatus": "InitialSilenceTimeout"}, + {"Offset": "5000000", "Duration": "25000000"}, + {}, + [], + None, +) + + +def _pcm16_wav(seconds: float) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(WAV_SAMPLE_RATE) + wav.writeframes(b"\x00\x00" * int(seconds * WAV_SAMPLE_RATE)) + return buffer.getvalue() @pytest.fixture(autouse=True) @@ -45,8 +67,8 @@ def azure_stt_price(monkeypatch: pytest.MonkeyPatch): ) -def _make_response(url: str) -> httpx.Response: - request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}) +def _make_response(url: str, uploaded: bytes = b"") -> httpx.Response: + request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}, content=uploaded) return httpx.Response(200, request=request, text=TRANSCRIPT) @@ -92,22 +114,13 @@ class TestAzureSpeechPassthroughHandler: assert logging_obj.model_call_details["custom_llm_provider"] == "azure_speech" assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) - @pytest.mark.parametrize( - "response_body", - [ - {"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}, - {"RecognitionStatus": "InitialSilenceTimeout"}, - {"Offset": "5000000", "Duration": "25000000"}, - {}, - [], - None, - ], - ) - def test_short_audio_without_recognized_duration_logs_zero_cost( - self, response_body: dict[str, object] | list[dict[str, object]] | None + @pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES) + @pytest.mark.parametrize("uploaded", [b"", b"not audio at all"]) + def test_short_audio_with_neither_recognized_nor_decodable_audio_logs_zero_cost( + self, response_body: dict[str, object] | list[dict[str, object]] | None, uploaded: bytes ): handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( - httpx_response=_make_response(SHORT_AUDIO_URL), + httpx_response=_make_response(SHORT_AUDIO_URL, uploaded), response_body=response_body, logging_obj=_make_logging_obj(), url_route=SHORT_AUDIO_URL, @@ -121,6 +134,60 @@ class TestAzureSpeechPassthroughHandler: assert handler_result["kwargs"]["model"] == "azure_speech/short-audio" assert handler_result["kwargs"]["response_cost"] == 0.0 + @pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES) + def test_short_audio_bills_the_uploaded_audio_when_nothing_was_recognized( + self, response_body: dict[str, object] | list[dict[str, object]] | None + ): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=2.0)), + response_body=response_body, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(2.0 * PRICE_PER_SECOND) + + @pytest.mark.parametrize( + "uploaded_seconds,expected_seconds", + [(1.0, TRANSCRIPT_AUDIO_SECONDS), (TRANSCRIPT_AUDIO_SECONDS + 2.0, TRANSCRIPT_AUDIO_SECONDS + 2.0)], + ) + def test_short_audio_bills_the_longer_of_uploaded_and_recognized_audio( + self, uploaded_seconds: float, expected_seconds: float + ): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=uploaded_seconds)), + response_body=TRANSCRIPT_BODY, + logging_obj=_make_logging_obj(), + url_route=SHORT_AUDIO_URL, + result=TRANSCRIPT, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_seconds * PRICE_PER_SECOND) + + def test_fast_transcription_ignores_the_uploaded_multipart_body(self): + handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(FAST_URL, _pcm16_wav(seconds=30.0)), + response_body=FAST_BODY, + logging_obj=_make_logging_obj(), + url_route=FAST_URL, + result=json.dumps(FAST_BODY), + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + ) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND) + @pytest.mark.parametrize( "response_body", [{"durationMilliseconds": 0}, {"durationMilliseconds": "5061"}, {"duration": 5061}, {}, [], None], diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9d210ac2f4f..636980eb6e3 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _proxy_general_settings, anthropic_proxy_route, azure_proxy_route, + azure_speech_proxy_route, bedrock_llm_proxy_route, bedrock_proxy_route, create_pass_through_route, @@ -6443,6 +6444,7 @@ AZURE_SPEECH_PCM16_HEADER: Final = ( b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00" ) AZURE_SPEECH_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + b"\x00" * 3072 +AZURE_SPEECH_WAV_SECONDS: Final = 3072 / (16000 * 2) AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range(256)) * 12 AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."} @@ -6826,6 +6828,79 @@ class TestAzureSpeechProxyRoute: assert "/azure_speech" in LiteLLMRoutes.mapped_pass_through_routes.value + def test_short_audio_with_no_recognized_speech_is_billed_for_the_uploaded_audio( + self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.integrations.custom_logger import CustomLogger + + class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.payloads.append(kwargs["standard_logging_object"]) + + recorder: Final = _Recorder() + monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder]) + monkeypatch.setitem( + litellm.model_cost, + "azure/speech/azure-stt", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": 0.25, + "output_cost_per_second": 0.0, + }, + ) + with respx.mock(assert_all_called=True) as upstream: + upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock( + return_value=httpx.Response(200, json={"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0}) + ) + + response = azure_speech_client.post( + f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", + content=AZURE_SPEECH_WAV_BYTES, + headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"}, + ) + + assert response.status_code == 200 + assert [p["model"] for p in recorder.payloads] == ["azure_speech/short-audio"] + assert recorder.payloads[0]["response_cost"] == pytest.approx(AZURE_SPEECH_WAV_SECONDS * 0.25) + + +class TestAzureSpeechProxyRoutePathTraversal: + """Calls the route function directly because httpx clients resolve dot segments before sending.""" + + @pytest.mark.parametrize( + "endpoint", + [ + f"speech/..{AZURE_SPEECH_BATCH_ENDPOINT}", + f"speech/recognition/../..{AZURE_SPEECH_BATCH_ENDPOINT}/", + f"speech/./..{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab", + ], + ) + @pytest.mark.asyncio + async def test_dot_segments_cannot_reach_shared_batch_resources_with_a_non_admin_key( + self, monkeypatch: pytest.MonkeyPatch, endpoint: str + ) -> None: + monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key") + monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus") + monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False) + request: Final = MagicMock(spec=Request) + request.method = "GET" + + with pytest.raises(HTTPException) as denied: + await azure_speech_proxy_route( + endpoint=endpoint, + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-virtual"), + ) + + assert denied.value.status_code == 403 + assert AZURE_SPEECH_FAST_ENDPOINT in str(denied.value.detail) + def _azure_speech_real_auth_attrs() -> dict[str, object]: from litellm.caching.caching import DualCache