diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5774e699ad7..443536ace88 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -11241,6 +11241,20 @@ "/v1/audio/transcriptions" ] }, + "azure/speech/MAI-Transcribe-2": { + "input_cost_per_second": 0.00002777777777777778, + "output_cost_per_second": 0.0, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://labs.ai.azure.com/innovations/mai-transcribe-2/" + }, + "azure/speech/MAI-Transcribe-1.5": { + "input_cost_per_second": 0.0001, + "output_cost_per_second": 0.0, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632" + }, "azure/tts-1": { "deprecation_date": "2026-12-15", "input_cost_per_character": 1.5e-05, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 9eef4f0ef9d..6a447e6b257 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1787,6 +1787,14 @@ async def azure_speech_proxy_route( ) raw_body: Final = await request.body() + if normalized_endpoint_path.endswith( + AZURE_SPEECH_FAST_TRANSCRIPTION_PATH + ) and HttpPassThroughEndpointHelpers.is_multipart(request): + async with request.form() as form: + definition: Final = form.get("definition") + if isinstance(definition, str): + setattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, {"definition": definition}) + endpoint_func: Final = create_pass_through_route( endpoint=endpoint, target=str(target_url), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py index d98b1c93a34..8fb5b88af1c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -4,6 +4,7 @@ from typing import Final from urllib.parse import urlparse import httpx +from pydantic import BaseModel, ConfigDict, Field, ValidationError from litellm._logging import verbose_proxy_logger from litellm.constants import ( @@ -26,9 +27,49 @@ from litellm.litellm_core_utils.litellm_logging import ( ) from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.types.utils import StandardPassThroughResponseObject +from litellm.utils import get_model_cost_key + + +class _EnhancedMode(BaseModel): + model_config = ConfigDict(frozen=True, strict=True) + + enabled: bool = False + model: str | None = Field(default=None, min_length=1) + + +class _FastTranscriptionDefinition(BaseModel): + model_config = ConfigDict(frozen=True, strict=True) + + enhanced_mode: _EnhancedMode | None = Field(default=None, alias="enhancedMode") class AzureSpeechPassthroughLoggingHandler: + @staticmethod + def _enhanced_transcription_model(url_route: str, request_body: Mapping[str, object]) -> str | None: + if not AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route): + return None + definition: Final = request_body.get("definition") + if not isinstance(definition, str): + return None + try: + parsed: Final = _FastTranscriptionDefinition.model_validate_json(definition) + except ValidationError: + return None + if parsed.enhanced_mode is None or not parsed.enhanced_mode.enabled: + return None + return parsed.enhanced_mode.model + + @staticmethod + def _resolve_model_and_pricing_model(url_route: str, request_body: Mapping[str, object]) -> tuple[str, str]: + enhanced_model: Final = AzureSpeechPassthroughLoggingHandler._enhanced_transcription_model( + url_route, request_body + ) + if enhanced_model is None: + return AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route), AZURE_SPEECH_PRICING_MODEL + model_name: Final = f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{enhanced_model}" + pricing_model: Final = get_model_cost_key(f"azure/speech/{enhanced_model}") + return model_name, pricing_model or AZURE_SPEECH_PRICING_MODEL + @staticmethod def _is_short_audio_route(url_route: str) -> bool: path: Final = urlparse(url_route).path @@ -99,6 +140,7 @@ class AzureSpeechPassthroughLoggingHandler: url_route: str, httpx_response: httpx.Response, response_body: Mapping[str, object] | Sequence[object] | None, + pricing_model: str = AZURE_SPEECH_PRICING_MODEL, ) -> float: audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds( url_route, httpx_response, response_body @@ -107,13 +149,13 @@ class AzureSpeechPassthroughLoggingHandler: return 0.0 try: prompt_cost, completion_cost = transcription_cost( - model=AZURE_SPEECH_PRICING_MODEL, + model=pricing_model, custom_llm_provider="azure", duration=audio_seconds, ) except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row verbose_proxy_logger.warning( - "No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e + "No price for %s, logging Azure Speech call at zero cost: %s", pricing_model, e ) return 0.0 return prompt_cost + completion_cost @@ -132,9 +174,11 @@ class AzureSpeechPassthroughLoggingHandler: **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler ) -> PassThroughEndpointLoggingTypedDict: try: - model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) + model_name, pricing_model = AzureSpeechPassthroughLoggingHandler._resolve_model_and_pricing_model( + url_route, request_body + ) response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost( - url_route, httpx_response, response_body + url_route, httpx_response, response_body, pricing_model ) updated_kwargs: Final = { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5774e699ad7..443536ace88 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11241,6 +11241,20 @@ "/v1/audio/transcriptions" ] }, + "azure/speech/MAI-Transcribe-2": { + "input_cost_per_second": 0.00002777777777777778, + "output_cost_per_second": 0.0, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://labs.ai.azure.com/innovations/mai-transcribe-2/" + }, + "azure/speech/MAI-Transcribe-1.5": { + "input_cost_per_second": 0.0001, + "output_cost_per_second": 0.0, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632" + }, "azure/tts-1": { "deprecation_date": "2026-12-15", "input_cost_per_character": 1.5e-05, diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py index a0d27e618f9..eac48a692dc 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -85,6 +85,66 @@ def _make_logging_obj() -> MagicMock: class TestAzureSpeechPassthroughHandler: + @pytest.mark.parametrize( + "definition,expected_model,model_rate", + [ + ('{"enhancedMode":{"enabled":true,"model":"MAI-Transcribe-2"}}', "MAI-Transcribe-2", 0.01), + ('{"enhancedMode":{"enabled":true,"model":"MAI-Transcribe-1.5"}}', "MAI-Transcribe-1.5", 0.02), + ('{"enhancedMode":{"enabled":true,"model":"future-model"}}', "future-model", None), + ('{"enhancedMode":{"enabled":false,"model":"MAI-Transcribe-2"}}', None, None), + ('{"enhancedMode":{"model":"MAI-Transcribe-2"}}', None, None), + ('{"enhancedMode":{"enabled":true}}', None, None), + ('{"enhancedMode":{"enabled":true,"model":""}}', None, None), + ('{"enhancedMode":{"enabled":"false","model":"MAI-Transcribe-2"}}', None, None), + ('{"enhancedMode":[]}', None, None), + ('{"enhancedMode":null}', None, None), + ('{"enhancedMode":{"enabled":true,"model":123}}', None, None), + ('{"locales":["en-US"]}', None, None), + ("not json", None, None), + ("[]", None, None), + (None, None, None), + ], + ) + def test_fast_transcription_logs_selected_model_and_uses_its_price_or_standard_fallback( + self, + monkeypatch: pytest.MonkeyPatch, + definition: str | None, + expected_model: str | None, + model_rate: float | None, + ) -> None: + if model_rate is not None: + monkeypatch.setitem( + litellm.model_cost, + f"azure/speech/{expected_model}", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": model_rate, + "output_cost_per_second": 0.0, + }, + ) + logging_obj: Final = _make_logging_obj() + handler_result: Final = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=_make_response(FAST_URL), + response_body=FAST_BODY, + logging_obj=logging_obj, + url_route=FAST_URL, + result=json.dumps(FAST_BODY), + start_time=datetime(2026, 10, 26), + end_time=datetime(2026, 10, 26), + cache_hit=False, + request_body={"definition": definition}, + ) + model: Final = f"azure_speech/{expected_model or 'fast-transcription'}" + cost: Final = FAST_AUDIO_SECONDS * (model_rate if model_rate is not None else PRICE_PER_SECOND) + + assert handler_result["kwargs"]["model"] == model + assert handler_result["kwargs"]["response_cost"] == pytest.approx(cost) + assert handler_result["kwargs"]["standard_logging_object"]["model"] == model + assert handler_result["kwargs"]["standard_logging_object"]["response_cost"] == pytest.approx(cost) + assert logging_obj.model_call_details["model"] == model + assert logging_obj.model_call_details["response_cost"] == pytest.approx(cost) + @pytest.mark.parametrize( "url_route,expected_model,expected_cost", [ diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 60043daa402..aa6ca6a78ef 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7700,6 +7700,13 @@ class TestAzureSpeechProxyRoute: def test_fast_transcription_multipart_upload_is_forwarded_byte_for_byte( self, azure_speech_client: TestClient ) -> None: + upload: Final = httpx.Request( + "POST", + f"http://testserver/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", + files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")}, + data={"definition": json.dumps({"locales": ["en-US"]})}, + ) + uploaded_body: Final = upload.read() with respx.mock(assert_all_called=True) as upstream: route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []}) @@ -7708,17 +7715,18 @@ class TestAzureSpeechProxyRoute: response = azure_speech_client.post( f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", params={"api-version": "2024-11-15"}, - files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")}, - data={"definition": json.dumps({"locales": ["en-US"]})}, - headers={"Authorization": "Bearer sk-virtual"}, + content=uploaded_body, + headers={ + "Authorization": "Bearer sk-virtual", + "Content-Type": upload.headers["content-type"], + }, ) assert response.status_code == 200 sent = route.calls.last.request - assert sent.headers["content-type"].startswith("multipart/form-data; boundary=") + assert sent.headers["content-type"] == upload.headers["content-type"] assert dict(sent.url.params) == {"api-version": "2024-11-15"} - assert AZURE_SPEECH_NON_UTF8_WAV_BYTES in sent.content - assert b'name="definition"' in sent.content + assert sent.content == uploaded_body assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key" assert "authorization" not in sent.headers @@ -7769,8 +7777,21 @@ class TestAzureSpeechProxyRoute: ("azure_speech/batch-transcription", "azure_speech", 0.0) ] + @pytest.mark.parametrize( + "enhanced_model,pricing_model", + [ + (None, "azure/speech/azure-stt"), + ("MAI-Transcribe-2", "azure/speech/MAI-Transcribe-2"), + ("MAI-Transcribe-1.5", "azure/speech/MAI-Transcribe-1.5"), + ("future-model", "azure/speech/azure-stt"), + ], + ) def test_fast_transcription_spend_is_priced_from_duration_milliseconds( - self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch + self, + azure_speech_client: TestClient, + monkeypatch: pytest.MonkeyPatch, + enhanced_model: str | None, + pricing_model: str, ) -> None: from litellm.integrations.custom_logger import CustomLogger @@ -7794,24 +7815,32 @@ class TestAzureSpeechProxyRoute: "output_cost_per_second": 0.0, }, ) + expected_rate: Final = litellm.model_cost[pricing_model]["input_cost_per_second"] + definition: Final = json.dumps( + {"locales": ["en-US"], "enhancedMode": {"enabled": True, "model": enhanced_model}} + if enhanced_model is not None + else {"locales": ["en-US"]} + ) with respx.mock(assert_all_called=True) as upstream: - upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock( - return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []}) - ) + route: Final = upstream.post( + f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}" + ).mock(return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []})) response = azure_speech_client.post( f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}", params={"api-version": "2024-11-15"}, files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")}, - data={"definition": json.dumps({"locales": ["en-US"]})}, + data={"definition": definition}, headers={"Authorization": "Bearer sk-virtual"}, ) assert response.status_code == 200 + assert definition.encode() in route.calls.last.request.content + assert AZURE_SPEECH_WAV_BYTES in route.calls.last.request.content assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [ - ("azure_speech/fast-transcription", "azure_speech") + (f"azure_speech/{enhanced_model or 'fast-transcription'}", "azure_speech") ] - assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * 0.25) + assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * expected_rate) def test_short_audio_spend_is_priced_from_the_recognized_duration( self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch