scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 51

fix(proxy): price Azure Speech fast transcription and limit unpriced batch writes to admins

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-17 18:14:04 +00:00
parent 1d8f19e4fd
commit 5f64dfd8dd
5 changed files with 220 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/constants.py
View file

@ -1574,13 +1574,17 @@ AZURE_SPEECH_CUSTOM_LLM_PROVIDER: Final = "azure_speech"
AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX: Final = "/azure_speech"
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX: Final = "/speech/"
AZURE_SPEECH_BATCH_PATH_PREFIX: Final = "/speechtotext/"
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH: Final = "/speechtotext/transcriptions:transcribe"
AZURE_SPEECH_UNPRICED_WRITE_METHODS: Final = frozenset({"POST", "PUT"})
AZURE_SPEECH_STT_DOMAIN: Final = "stt.speech.microsoft.com"
AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN: Final = "api.cognitive.microsoft.com"
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER: Final = "Ocp-Apim-Subscription-Key"
AZURE_SPEECH_SHORT_AUDIO_MODEL: Final = "short-audio"
AZURE_SPEECH_BATCH_MODEL: Final = "batch-transcription"
AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL: Final = "fast-transcription"
AZURE_SPEECH_PRICING_MODEL: Final = "azure/speech/azure-stt"
AZURE_SPEECH_TICKS_PER_SECOND: Final = 10_000_000
AZURE_SPEECH_MILLISECONDS_PER_SECOND: Final = 1_000
BASE_MCP_ROUTE: Final = "/mcp"

22
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -33,10 +33,12 @@ from litellm.constants import (
AZURE_SPEECH_BATCH_PATH_PREFIX,
AZURE_SPEECH_COGNITIVE_SERVICES_DOMAIN,
AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH,
AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX,
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX,
AZURE_SPEECH_STT_DOMAIN,
AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER,
AZURE_SPEECH_UNPRICED_WRITE_METHODS,
BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES,
)
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
@ -65,6 +67,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
get_request_body,
is_json_content_type,
)
from litellm.proxy.common_utils.resource_ownership import is_proxy_admin
from litellm.proxy.common_utils.sse_keepalive import (
wrap_passthrough_sse_bytes_with_keepalive_pings,
)
@ -1357,6 +1360,14 @@ def resolve_azure_speech_base_url(endpoint_path: str, api_base: str | None, regi
return httpx.URL(f"https://{region}.{domain}")
def azure_speech_write_is_unpriced(method: str, endpoint_path: str) -> bool:
return (
endpoint_path.startswith(AZURE_SPEECH_BATCH_PATH_PREFIX)
and endpoint_path != AZURE_SPEECH_FAST_TRANSCRIPTION_PATH
and method.upper() in AZURE_SPEECH_UNPRICED_WRITE_METHODS
)
@router.api_route(
f"{AZURE_SPEECH_PASS_THROUGH_ROUTE_PREFIX}/{{endpoint:path}}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: fastapi route methods must be a list
@ -1395,6 +1406,17 @@ async def azure_speech_proxy_route(
"AZURE_SPEECH_REGION or AZURE_SPEECH_API_BASE in the proxy environment."
),
)
if azure_speech_write_is_unpriced(
method=request.method, endpoint_path=normalized_endpoint_path
) and not is_proxy_admin(user_api_key_dict):
raise HTTPException(
status_code=403,
detail=(
f"{request.method} {normalized_endpoint_path} creates Azure Speech work whose cost is unknown at "
"request time, so it is limited to proxy admin keys. Use "
f"{AZURE_SPEECH_FAST_TRANSCRIPTION_PATH} for transcription that is priced per request."
),
)
azure_speech_api_key: Final = passthrough_endpoint_router.get_credentials(
custom_llm_provider=AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
region_name=None,

30
litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py
View file

@ -9,6 +9,9 @@ from litellm._logging import verbose_proxy_logger
from litellm.constants import (
AZURE_SPEECH_BATCH_MODEL,
AZURE_SPEECH_CUSTOM_LLM_PROVIDER,
AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL,
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH,
AZURE_SPEECH_MILLISECONDS_PER_SECOND,
AZURE_SPEECH_PRICING_MODEL,
AZURE_SPEECH_SHORT_AUDIO_MODEL,
AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX,
@ -28,10 +31,16 @@ class AzureSpeechPassthroughLoggingHandler:
def _is_short_audio_route(url_route: str) -> bool:
return urlparse(url_route).path.startswith(AZURE_SPEECH_SHORT_AUDIO_PATH_PREFIX)
@staticmethod
def _is_fast_transcription_route(url_route: str) -> bool:
return urlparse(url_route).path.endswith(AZURE_SPEECH_FAST_TRANSCRIPTION_PATH)
@staticmethod
def _model_from_url_route(url_route: str) -> str:
if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route):
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_SHORT_AUDIO_MODEL}"
if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route):
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL}"
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}"
@staticmethod
@ -45,10 +54,25 @@ class AzureSpeechPassthroughLoggingHandler:
return (offset + duration) / AZURE_SPEECH_TICKS_PER_SECOND
@staticmethod
def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float:
if not AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route):
def _fast_transcription_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float:
if not isinstance(response_body, Mapping):
return 0.0
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body)
duration_milliseconds: Final = response_body.get("durationMilliseconds")
if not isinstance(duration_milliseconds, int):
return 0.0
return duration_milliseconds / AZURE_SPEECH_MILLISECONDS_PER_SECOND
@staticmethod
def _billed_audio_seconds(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float:
if AzureSpeechPassthroughLoggingHandler._is_short_audio_route(url_route):
return AzureSpeechPassthroughLoggingHandler._recognized_audio_seconds(response_body)
if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route):
return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body)
return 0.0
@staticmethod
def _response_cost(url_route: str, response_body: Mapping[str, object] | Sequence[object] | None) -> float:
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(url_route, response_body)
if audio_seconds <= 0.0:
return 0.0
try:

39
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py
View file

@ -13,9 +13,19 @@ from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
SHORT_AUDIO_URL = "https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US"
SHORT_AUDIO_URL = (
"https://eastus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=en-US"
)
BATCH_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2/transcriptions"
TRANSCRIPT_BODY = {"RecognitionStatus": "Success", "Offset": 5000000, "Duration": 25000000, "DisplayText": "Hello world."}
FAST_URL = "https://eastus.api.cognitive.microsoft.com/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
FAST_BODY = {"durationMilliseconds": 5061, "combinedPhrases": [{"text": "Hello world."}]}
FAST_AUDIO_SECONDS = 5.061
TRANSCRIPT_BODY = {
"RecognitionStatus": "Success",
"Offset": 5000000,
"Duration": 25000000,
"DisplayText": "Hello world.",
}
TRANSCRIPT = json.dumps(TRANSCRIPT_BODY)
TRANSCRIPT_AUDIO_SECONDS = 3.0
PRICE_PER_SECOND = 0.5
@ -52,6 +62,7 @@ class TestAzureSpeechPassthroughHandler:
"url_route,expected_model,expected_cost",
[
(SHORT_AUDIO_URL, "azure_speech/short-audio", TRANSCRIPT_AUDIO_SECONDS * PRICE_PER_SECOND),
(FAST_URL, "azure_speech/fast-transcription", FAST_AUDIO_SECONDS * PRICE_PER_SECOND),
(BATCH_URL, "azure_speech/batch-transcription", 0.0),
(f"{BATCH_URL}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab/files", "azure_speech/batch-transcription", 0.0),
],
@ -61,7 +72,7 @@ class TestAzureSpeechPassthroughHandler:
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(url_route),
response_body=TRANSCRIPT_BODY,
response_body={**TRANSCRIPT_BODY, **FAST_BODY},
logging_obj=logging_obj,
url_route=url_route,
result=TRANSCRIPT,
@ -110,6 +121,28 @@ class TestAzureSpeechPassthroughHandler:
assert handler_result["kwargs"]["model"] == "azure_speech/short-audio"
assert handler_result["kwargs"]["response_cost"] == 0.0
@pytest.mark.parametrize(
"response_body",
[{"durationMilliseconds": 0}, {"durationMilliseconds": "5061"}, {"duration": 5061}, {}, [], None],
)
def test_fast_transcription_without_duration_milliseconds_logs_zero_cost(
self, response_body: dict[str, object] | list[dict[str, object]] | None
):
handler_result = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(FAST_URL),
response_body=response_body,
logging_obj=_make_logging_obj(),
url_route=FAST_URL,
result="",
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={},
)
assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription"
assert handler_result["kwargs"]["response_cost"] == 0.0
def test_missing_price_entry_still_logs_the_row_at_zero_cost(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.delitem(litellm.model_cost, "azure/speech/azure-stt")

146
tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -6141,6 +6141,7 @@ class TestAzureRelayDeploymentSegment:
AZURE_SPEECH_SHORT_AUDIO_ENDPOINT: Final = "/speech/recognition/conversation/cognitiveservices/v1"
AZURE_SPEECH_BATCH_ENDPOINT: Final = "/speechtotext/v3.2/transcriptions"
AZURE_SPEECH_FAST_ENDPOINT: Final = "/speechtotext/transcriptions:transcribe"
AZURE_SPEECH_PCM16_HEADER: Final = (
b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00"
)
@ -6149,8 +6150,7 @@ AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range
AZURE_SPEECH_TRANSCRIPT: Final = {"RecognitionStatus": "Success", "DisplayText": "The eagle has landed."}
@pytest.fixture
def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
def _azure_speech_test_client(monkeypatch: pytest.MonkeyPatch, caller: UserAPIKeyAuth) -> TestClient:
from litellm.proxy.proxy_server import app
monkeypatch.setenv("AZURE_SPEECH_API_KEY", "server-subscription-key")
@ -6159,8 +6159,20 @@ def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
yield TestClient(app)
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: caller)
return TestClient(app)
@pytest.fixture
def azure_speech_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
yield _azure_speech_test_client(monkeypatch, UserAPIKeyAuth(api_key="sk-virtual"))
@pytest.fixture
def azure_speech_admin_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
yield _azure_speech_test_client(
monkeypatch, UserAPIKeyAuth(api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN)
)
class TestAzureSpeechProxyRoute:
@ -6193,17 +6205,19 @@ class TestAzureSpeechProxyRoute:
assert "authorization" not in sent.headers
assert "caller-supplied-key" not in repr(sent.headers)
def test_batch_json_goes_to_the_cognitive_services_host(self, azure_speech_client: TestClient) -> None:
def test_admin_batch_job_creation_goes_to_the_cognitive_services_host(
self, azure_speech_admin_client: TestClient
) -> None:
body: Final = {"contentUrls": ["https://example.com/a.wav"], "locale": "en-US", "displayName": "job"}
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock(
return_value=httpx.Response(201, json={"self": "https://eastus.api.cognitive.microsoft.com/x"})
)
response = azure_speech_client.post(
response = azure_speech_admin_client.post(
f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}",
json=body,
headers={"Authorization": "Bearer sk-virtual"},
headers={"Authorization": "Bearer sk-admin"},
)
assert response.status_code == 201
@ -6212,22 +6226,80 @@ class TestAzureSpeechProxyRoute:
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
assert "authorization" not in sent.headers
def test_batch_multipart_upload_is_forwarded_byte_for_byte(self, azure_speech_client: TestClient) -> None:
@pytest.mark.parametrize(
"method,endpoint",
[
("POST", AZURE_SPEECH_BATCH_ENDPOINT),
("POST", "/speechtotext/v3.2/models"),
("PUT", "/speechtotext/v3.2/endpoints/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"),
],
)
def test_non_admin_key_cannot_create_unpriced_batch_work(
self, azure_speech_client: TestClient, method: str, endpoint: str
) -> None:
with respx.mock(assert_all_called=False) as upstream:
catch_all = upstream.route().mock(return_value=httpx.Response(201, json={"status": "NotStarted"}))
response = azure_speech_client.request(
method,
f"/azure_speech{endpoint}",
json={"contentUrls": ["https://example.com/a.wav"], "locale": "en-US"},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 403, response.text
assert AZURE_SPEECH_FAST_ENDPOINT in response.text
assert not catch_all.called
def test_non_admin_key_can_still_read_delete_and_fast_transcribe_in_the_batch_family(
self, azure_speech_client: TestClient
) -> None:
job_path: Final = f"{AZURE_SPEECH_BATCH_ENDPOINT}/8a5d3f2c-0b1e-4c7d-9e6f-1234567890ab"
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_BATCH_ENDPOINT}").mock(
return_value=httpx.Response(201, json={"status": "NotStarted"})
upstream.get(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock(
return_value=httpx.Response(200, json={"status": "Succeeded"})
)
upstream.delete(f"https://eastus.api.cognitive.microsoft.com{job_path}").mock(
return_value=httpx.Response(204)
)
upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []})
)
statuses = [
azure_speech_client.get(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}),
azure_speech_client.delete(f"/azure_speech{job_path}", headers={"Authorization": "Bearer sk-virtual"}),
azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
),
]
assert [r.status_code for r in statuses] == [200, 204, 200]
def test_fast_transcription_multipart_upload_is_forwarded_byte_for_byte(
self, azure_speech_client: TestClient
) -> None:
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []})
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}",
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 201
assert response.status_code == 200
sent = route.calls.last.request
assert sent.headers["content-type"].startswith("multipart/form-data; boundary=")
assert dict(sent.url.params) == {"api-version": "2024-11-15"}
assert AZURE_SPEECH_NON_UTF8_WAV_BYTES in sent.content
assert b'name="definition"' in sent.content
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
@ -6247,7 +6319,7 @@ class TestAzureSpeechProxyRoute:
@pytest.mark.parametrize("method", ["GET", "POST"])
def test_batch_requests_are_logged_as_azure_speech_not_assemblyai(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch, method: str
self, azure_speech_admin_client: TestClient, monkeypatch: pytest.MonkeyPatch, method: str
) -> None:
from litellm.integrations.custom_logger import CustomLogger
@ -6266,11 +6338,11 @@ class TestAzureSpeechProxyRoute:
return_value=httpx.Response(200, json={"values": []})
)
response = azure_speech_client.request(
response = azure_speech_admin_client.request(
method,
f"/azure_speech{AZURE_SPEECH_BATCH_ENDPOINT}",
json={"locale": "en-US"} if method == "POST" else None,
headers={"Authorization": "Bearer sk-virtual"},
headers={"Authorization": "Bearer sk-admin"},
)
assert response.status_code == 200
@ -6278,6 +6350,50 @@ class TestAzureSpeechProxyRoute:
("azure_speech/batch-transcription", "azure_speech", 0.0)
]
def test_fast_transcription_spend_is_priced_from_duration_milliseconds(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.integrations.custom_logger import CustomLogger
class _Recorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.payloads: list[dict[str, object]] = [] # mutable-ok: test recorder accumulates callback payloads
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
self.payloads.append(kwargs["standard_logging_object"])
recorder: Final = _Recorder()
monkeypatch.setattr(litellm, "_async_success_callback", [*litellm._async_success_callback, recorder])
monkeypatch.setitem(
litellm.model_cost,
"azure/speech/azure-stt",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": 0.25,
"output_cost_per_second": 0.0,
},
)
with respx.mock(assert_all_called=True) as upstream:
upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []})
)
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [
("azure_speech/fast-transcription", "azure_speech")
]
assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * 0.25)
def test_short_audio_spend_is_priced_from_the_recognized_duration(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
) -> None: