mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
fix(proxy): canonicalize azure speech paths and bill uploaded short audio
Resolve dot segments in the /azure_speech endpoint path before the endpoint family and the admin-only batch guard are decided, so the guard and the forwarded upstream path agree. Bill short-audio requests for the longer of the uploaded audio duration and the recognized duration, so a NoMatch or silence response still charges for the audio Azure processed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin
parent
1e6b33ffab
commit
1e7c5400fd
4 changed files with 207 additions and 23 deletions
|
|
@ -12,6 +12,7 @@ import hmac
|
|||
import inspect
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -1408,6 +1409,18 @@ def azure_speech_path_manages_shared_resources(endpoint_path: str) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def canonical_azure_speech_endpoint_path(endpoint: str) -> str:
|
||||
"""
|
||||
The path Azure will actually serve, with ``.`` and ``..`` segments resolved, so the
|
||||
endpoint family and the admin guard are decided on the same path the upstream request uses.
|
||||
"""
|
||||
raw_path: Final = httpx.URL(endpoint).path
|
||||
resolved_path: Final = posixpath.normpath(f"/{raw_path.lstrip('/')}")
|
||||
if raw_path.endswith("/") and resolved_path != "/":
|
||||
return f"{resolved_path}/"
|
||||
return resolved_path
|
||||
|
||||
|
||||
@router.api_route(
|
||||
f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}",
|
||||
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list
|
||||
|
|
@ -1430,8 +1443,7 @@ async def azure_speech_proxy_route(
|
|||
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/azure_speech)
|
||||
"""
|
||||
endpoint_path: Final = httpx.URL(endpoint).path
|
||||
normalized_endpoint_path: Final = endpoint_path if endpoint_path.startswith("/") else f"/{endpoint_path}"
|
||||
normalized_endpoint_path: Final = canonical_azure_speech_endpoint_path(endpoint)
|
||||
base_url: Final = resolve_azure_speech_base_url(
|
||||
endpoint_path=normalized_endpoint_path,
|
||||
api_base=get_secret_str(secret_name="AZURE_SPEECH_API_BASE"),
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.constants import (
|
|||
AZURE_SPEECH_TICKS_PER_SECOND,
|
||||
)
|
||||
from litellm.cost_calculator import transcription_cost
|
||||
from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload,
|
||||
|
|
@ -53,6 +54,23 @@ class AzureSpeechPassthroughLoggingHandler:
|
|||
return 0.0
|
||||
return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND
|
||||
|
||||
@staticmethod
|
||||
def _uploaded_audio_seconds(httpx_response: httpx.Response) -> float:
|
||||
try:
|
||||
uploaded_audio: Final = httpx_response.request.content
|
||||
except RuntimeError:
|
||||
return 0.0
|
||||
return calculate_request_duration(uploaded_audio) or 0.0
|
||||
|
||||
@staticmethod
|
||||
def _short_audio_seconds(
|
||||
httpx_response: httpx.Response, response_body: Mapping[str, object] | Sequence[object] | None
|
||||
) -> float:
|
||||
return max(
|
||||
AzureSpeechPassthroughLoggingHandler._uploaded_audio_seconds(httpx_response),
|
||||
AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fast_transcription_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float:
|
||||
if not isinstance(response_body, Mapping):
|
||||
|
|
@ -63,16 +81,26 @@ class AzureSpeechPassthroughLoggingHandler:
|
|||
return duration_milliseconds / AZURE_SPEECH_MILLISECONDS_PER_SECOND
|
||||
|
||||
@staticmethod
|
||||
def _billed_audio_seconds(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float:
|
||||
def _billed_audio_seconds(
|
||||
url_route: str,
|
||||
httpx_response: httpx.Response,
|
||||
response_body: Mapping[str, object] | Sequence[object] | None,
|
||||
) -> float:
|
||||
if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route):
|
||||
return AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body)
|
||||
return AzureSpeechPassthroughLoggingHandler._short_audio_seconds(httpx_response, response_body)
|
||||
if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route):
|
||||
return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body)
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float:
|
||||
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(url_route, response_body)
|
||||
def _response_cost(
|
||||
url_route: str,
|
||||
httpx_response: httpx.Response,
|
||||
response_body: Mapping[str, object] | Sequence[object] | None,
|
||||
) -> float:
|
||||
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(
|
||||
url_route, httpx_response, response_body
|
||||
)
|
||||
if audio_seconds <= 0.0:
|
||||
return 0.0
|
||||
try:
|
||||
|
|
@ -103,7 +131,9 @@ class AzureSpeechPassthroughLoggingHandler:
|
|||
) -> PassThroughEndpointLoggingTypedDict:
|
||||
try:
|
||||
model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route)
|
||||
response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(url_route, response_body)
|
||||
response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(
|
||||
url_route, httpx_response, response_body
|
||||
)
|
||||
|
||||
updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict
|
||||
**kwargs,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import io
|
||||
import json
|
||||
import wave
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -29,6 +32,25 @@ TRANSCRIPT_BODY = {
|
|||
TRANSCRIPT = json.dumps(TRANSCRIPT_BODY)
|
||||
TRANSCRIPT_AUDIO_SECONDS = 3.0
|
||||
PRICE_PER_SECOND = 0.5
|
||||
WAV_SAMPLE_RATE: Final = 16000
|
||||
UNRECOGNIZED_BODIES: Final = (
|
||||
{"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0},
|
||||
{"RecognitionStatus": "InitialSilenceTimeout"},
|
||||
{"Offset": "5000000", "Duration": "25000000"},
|
||||
{},
|
||||
[],
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _pcm16_wav(seconds: float) -> bytes:
|
||||
buffer: Final = io.BytesIO()
|
||||
with wave.open(buffer, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(WAV_SAMPLE_RATE)
|
||||
wav.writeframes(b"\x00\x00" * int(seconds * WAV_SAMPLE_RATE))
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -45,8 +67,8 @@ def azure_stt_price(monkeypatch: pytest.MonkeyPatch):
|
|||
)
|
||||
|
||||
|
||||
def _make_response(url: str) -> httpx.Response:
|
||||
request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"})
|
||||
def _make_response(url: str, uploaded: bytes = b"") -> httpx.Response:
|
||||
request = httpx.Request("POST", url, headers={"Ocp-Apim-Subscription-Key": "server-secret"}, content=uploaded)
|
||||
return httpx.Response(200, request=request, text=TRANSCRIPT)
|
||||
|
||||
|
||||
|
|
@ -92,22 +114,13 @@ class TestAzureSpeechPassthroughHandler:
|
|||
assert logging_obj.model_call_details["custom_llm_provider"] == "azure_speech"
|
||||
assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_body",
|
||||
[
|
||||
{"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0},
|
||||
{"RecognitionStatus": "InitialSilenceTimeout"},
|
||||
{"Offset": "5000000", "Duration": "25000000"},
|
||||
{},
|
||||
[],
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_short_audio_without_recognized_duration_logs_zero_cost(
|
||||
self, response_body: dict[str, object] | list[dict[str, object]] | None
|
||||
@pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES)
|
||||
@pytest.mark.parametrize("uploaded", [b"", b"not audio at all"])
|
||||
def test_short_audio_with_neither_recognized_nor_decodable_audio_logs_zero_cost(
|
||||
self, response_body: dict[str, object] | list[dict[str, object]] | None, uploaded: bytes
|
||||
):
|
||||
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
|
||||
httpx_response=_make_response(SHORT_AUDIO_URL),
|
||||
httpx_response=_make_response(SHORT_AUDIO_URL, uploaded),
|
||||
response_body=response_body,
|
||||
logging_obj=_make_logging_obj(),
|
||||
url_route=SHORT_AUDIO_URL,
|
||||
|
|
@ -121,6 +134,60 @@ class TestAzureSpeechPassthroughHandler:
|
|||
assert handler_result["kwargs"]["model"] == "azure_speech/short-audio"
|
||||
assert handler_result["kwargs"]["response_cost"] == 0.0
|
||||
|
||||
@pytest.mark.parametrize("response_body", UNRECOGNIZED_BODIES)
|
||||
def test_short_audio_bills_the_uploaded_audio_when_nothing_was_recognized(
|
||||
self, response_body: dict[str, object] | list[dict[str, object]] | None
|
||||
):
|
||||
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
|
||||
httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=2.0)),
|
||||
response_body=response_body,
|
||||
logging_obj=_make_logging_obj(),
|
||||
url_route=SHORT_AUDIO_URL,
|
||||
result="",
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
)
|
||||
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(2.0 * PRICE_PER_SECOND)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"uploaded_seconds,expected_seconds",
|
||||
[(1.0, TRANSCRIPT_AUDIO_SECONDS), (TRANSCRIPT_AUDIO_SECONDS + 2.0, TRANSCRIPT_AUDIO_SECONDS + 2.0)],
|
||||
)
|
||||
def test_short_audio_bills_the_longer_of_uploaded_and_recognized_audio(
|
||||
self, uploaded_seconds: float, expected_seconds: float
|
||||
):
|
||||
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
|
||||
httpx_response=_make_response(SHORT_AUDIO_URL, _pcm16_wav(seconds=uploaded_seconds)),
|
||||
response_body=TRANSCRIPT_BODY,
|
||||
logging_obj=_make_logging_obj(),
|
||||
url_route=SHORT_AUDIO_URL,
|
||||
result=TRANSCRIPT,
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
)
|
||||
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_seconds * PRICE_PER_SECOND)
|
||||
|
||||
def test_fast_transcription_ignores_the_uploaded_multipart_body(self):
|
||||
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
|
||||
httpx_response=_make_response(FAST_URL, _pcm16_wav(seconds=30.0)),
|
||||
response_body=FAST_BODY,
|
||||
logging_obj=_make_logging_obj(),
|
||||
url_route=FAST_URL,
|
||||
result=json.dumps(FAST_BODY),
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
cache_hit=False,
|
||||
request_body={},
|
||||
)
|
||||
|
||||
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_body",
|
||||
[{"durationMilliseconds": 0}, {"durationMilliseconds": "5061"}, {"duration": 5061}, {}, [], None],
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
|||
_proxy_general_settings,
|
||||
anthropic_proxy_route,
|
||||
azure_proxy_route,
|
||||
azure_speech_proxy_route,
|
||||
bedrock_llm_proxy_route,
|
||||
bedrock_proxy_route,
|
||||
create_pass_through_route,
|
||||
|
|
@ -6443,6 +6444,7 @@ AZURE_SPEECH_PCM16_HEADER: Final = (
|
|||
b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00"
|
||||
)
|
||||
AZURE_SPEECH_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + b"\x00" * 3072
|
||||
AZURE_SPEECH_WAV_SECONDS: Final = 3072 / (16000 * 2)
|
||||
AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range(256)) * 12
|
||||
AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."}
|
||||
|
||||
|
|
@ -6826,6 +6828,79 @@ class TestAzureSpeechProxyRoute:
|
|||
|
||||
assert "/azure_speech" in LiteLLMRoutes.mapped_pass_through_routes.value
|
||||
|
||||
def test_short_audio_with_no_recognized_speech_is_billed_for_the_uploaded_audio(
|
||||
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
class _Recorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
|
||||
self.payloads.append(kwargs["standard_logging_object"])
|
||||
|
||||
recorder: Final = _Recorder()
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder])
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"azure/speech/azure-stt",
|
||||
{
|
||||
"litellm_provider": "azure",
|
||||
"mode": "audio_transcription",
|
||||
"input_cost_per_second": 0.25,
|
||||
"output_cost_per_second": 0.0,
|
||||
},
|
||||
)
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock(
|
||||
return_value=httpx.Response(200, json={"RecognitionStatus": "NoMatch", "Offset": 0, "Duration": 0})
|
||||
)
|
||||
|
||||
response = azure_speech_client.post(
|
||||
f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}",
|
||||
content=AZURE_SPEECH_WAV_BYTES,
|
||||
headers={"Content-Type": "audio/wav", "Authorization": "Bearer sk-virtual"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [p["model"] for p in recorder.payloads] == ["azure_speech/short-audio"]
|
||||
assert recorder.payloads[0]["response_cost"] == pytest.approx(AZURE_SPEECH_WAV_SECONDS * 0.25)
|
||||
|
||||
|
||||
class TestAzureSpeechProxyRoutePathTraversal:
|
||||
"""Calls the route function directly because httpx clients resolve dot segments before sending."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint",
|
||||
[
|
||||
f"speech/..{AZURE_SPEECH_BATCH_ENDPOINT}",
|
||||
f"speech/recognition/../..{AZURE_SPEECH_BATCH_ENDPOINT}/",
|
||||
f"speech/./..{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab",
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_dot_segments_cannot_reach_shared_batch_resources_with_a_non_admin_key(
|
||||
self, monkeypatch: pytest.MonkeyPatch, endpoint: str
|
||||
) -> None:
|
||||
monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key")
|
||||
monkeypatch.setenv("AZURE_SPEECH_REGION", "eastus")
|
||||
monkeypatch.delenv("AZURE_SPEECH_API_BASE", raising=False)
|
||||
request: Final = MagicMock(spec=Request)
|
||||
request.method = "GET"
|
||||
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
await azure_speech_proxy_route(
|
||||
endpoint=endpoint,
|
||||
request=request,
|
||||
fastapi_response=MagicMock(spec=Response),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-virtual"),
|
||||
)
|
||||
|
||||
assert denied.value.status_code == 403
|
||||
assert AZURE_SPEECH_FAST_ENDPOINT in str(denied.value.detail)
|
||||
|
||||
|
||||
def _azure_speech_real_auth_attrs() -> dict[str, object]:
|
||||
from litellm.caching.caching import DualCache
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue