diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 95adfda8c49..bb7f9a5af0b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12084,6 +12084,17 @@ "mode": "audio_speech", "source": "https://azure.microsoft.com/en-us/pricing/calculator/" }, + "azure/speech/MAI-Transcribe-2": { + "audio_transcription_config": "azure_speech", + "input_cost_per_second": 0.00015, + "litellm_provider": "azure", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, "azure/speech/azure-stt": { "audio_transcription_config": "azure_speech", "input_cost_per_second": 0.0002777778, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py index d98b1c93a34..2d436cd341e 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py @@ -1,3 +1,5 @@ +import json +import re from collections.abc import Mapping, Sequence from datetime import datetime from typing import Final @@ -46,6 +48,45 @@ class AzureSpeechPassthroughLoggingHandler: return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL}" return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}" + @staticmethod + def _definition_from_multipart_body(httpx_response: httpx.Response) -> str | None: + content_type: Final = httpx_response.request.headers.get("content-type", "") + boundary_match: Final = re.search(r"boundary=(?P[^;]+)", content_type) + if boundary_match is None: + return None + boundary: Final = boundary_match.group("boundary").strip().strip('"') + try: + request_content: Final = httpx_response.request.content + except RuntimeError: + return None + definition_pattern: Final = re.compile( + rb'Content-Disposition: form-data; name="definition"\r\n\r\n(?P.*?)\r\n--' + + re.escape(boundary.encode()), + re.DOTALL, + ) + definition_match: Final = definition_pattern.search(request_content) + if definition_match is None: + return None + return definition_match.group("definition").decode("utf-8", errors="ignore") + + @staticmethod + def _enhanced_mode_model(request_body: Mapping[str, object], httpx_response: httpx.Response) -> str | None: + definition: object = request_body.get("definition") + if definition is None: + definition = AzureSpeechPassthroughLoggingHandler._definition_from_multipart_body(httpx_response) + if isinstance(definition, str): + try: + definition = json.loads(definition) + except json.JSONDecodeError: + return None + if not isinstance(definition, Mapping): + return None + enhanced_mode: Final = definition.get("enhancedMode") + if not isinstance(enhanced_mode, Mapping) or enhanced_mode.get("enabled") is not True: + return None + model: Final = enhanced_mode.get("model") + return model if isinstance(model, str) and model else None + @staticmethod def _recognized_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float: if not isinstance(response_body, Mapping): @@ -94,29 +135,48 @@ class AzureSpeechPassthroughLoggingHandler: return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body) return 0.0 + @staticmethod + def _price_with_model(pricing_model: str, audio_seconds: float) -> float: + prompt_cost, completion_cost = transcription_cost( + model=pricing_model, + custom_llm_provider="azure", + duration=audio_seconds, + ) + return prompt_cost + completion_cost + @staticmethod def _response_cost( url_route: str, httpx_response: httpx.Response, response_body: Mapping[str, object] | Sequence[object] | None, + enhanced_mode_model: str | None = None, ) -> float: audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds( url_route, httpx_response, response_body ) if audio_seconds <= 0.0: return 0.0 + pricing_model: Final = ( + f"azure/speech/{enhanced_mode_model}" if enhanced_mode_model else AZURE_SPEECH_PRICING_MODEL + ) try: - prompt_cost, completion_cost = transcription_cost( - model=AZURE_SPEECH_PRICING_MODEL, - custom_llm_provider="azure", - duration=audio_seconds, + return AzureSpeechPassthroughLoggingHandler._price_with_model(pricing_model, audio_seconds) + except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row + if pricing_model == AZURE_SPEECH_PRICING_MODEL: + verbose_proxy_logger.warning( + "No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e + ) + return 0.0 + verbose_proxy_logger.warning( + "No price for %s, falling back to %s: %s", pricing_model, AZURE_SPEECH_PRICING_MODEL, e ) + try: + return AzureSpeechPassthroughLoggingHandler._price_with_model(AZURE_SPEECH_PRICING_MODEL, audio_seconds) except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row verbose_proxy_logger.warning( "No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e ) return 0.0 - return prompt_cost + completion_cost @staticmethod def azure_speech_passthrough_handler( @@ -132,9 +192,18 @@ class AzureSpeechPassthroughLoggingHandler: **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler ) -> PassThroughEndpointLoggingTypedDict: try: - model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) + enhanced_mode_model: Final = ( + AzureSpeechPassthroughLoggingHandler._enhanced_mode_model(request_body, httpx_response) + if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route) + else None + ) + model_name: Final = ( + f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{enhanced_mode_model}" + if enhanced_mode_model + else AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route) + ) response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost( - url_route, httpx_response, response_body + url_route, httpx_response, response_body, enhanced_mode_model ) updated_kwargs: Final = { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 95adfda8c49..bb7f9a5af0b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12084,6 +12084,17 @@ "mode": "audio_speech", "source": "https://azure.microsoft.com/en-us/pricing/calculator/" }, + "azure/speech/MAI-Transcribe-2": { + "audio_transcription_config": "azure_speech", + "input_cost_per_second": 0.00015, + "litellm_provider": "azure", + "mode": "audio_transcription", + "output_cost_per_second": 0.0, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ] + }, "azure/speech/azure-stt": { "audio_transcription_config": "azure_speech", "input_cost_per_second": 0.0002777778, diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py index a0d27e618f9..a913c33129a 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py @@ -279,6 +279,109 @@ class TestIsAzureSpeechRoute: assert "response_cost" not in normalized["kwargs"] +MAI_MODEL = "MAI-Transcribe-2" +MAI_PRICE_PER_SECOND = 0.25 +MULTIPART_BOUNDARY = "litellm-test-boundary" + + +def _make_multipart_response(url: str, definition: str | None) -> httpx.Response: + parts: Final = ( + f"--{MULTIPART_BOUNDARY}\r\n" + 'Content-Disposition: form-data; name="audio"; filename="audio.wav"\r\n' + "Content-Type: audio/wav\r\n\r\n" + ) + definition_part: Final = ( + f'--{MULTIPART_BOUNDARY}\r\nContent-Disposition: form-data; name="definition"\r\n\r\n{definition}\r\n' + if definition is not None + else "" + ) + body: Final = f"{parts}FAKE-AUDIO\r\n{definition_part}--{MULTIPART_BOUNDARY}--\r\n".encode() + request = httpx.Request( + "POST", + url, + headers={"Content-Type": f"multipart/form-data; boundary={MULTIPART_BOUNDARY}"}, + content=body, + ) + return httpx.Response(200, request=request, text=json.dumps(FAST_BODY)) + + +def _definition_with_model(model: str, enabled: bool = True) -> str: + return json.dumps({"enhancedMode": {"enabled": enabled, "model": model}}) + + +class TestAzureSpeechEnhancedModePricing: + @pytest.fixture(autouse=True) + def mai_price(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setitem( + litellm.model_cost, + f"azure/speech/{MAI_MODEL}", + { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": MAI_PRICE_PER_SECOND, + "output_cost_per_second": 0.0, + }, + ) + + def _run_handler( + self, httpx_response: httpx.Response, request_body: dict[str, object] | None = None + ) -> dict[str, object]: + return AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler( + httpx_response=httpx_response, + response_body=FAST_BODY, + logging_obj=_make_logging_obj(), + url_route=FAST_URL, + result=json.dumps(FAST_BODY), + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body=request_body or {}, + ) + + def test_fast_transcription_with_enhanced_mode_model_in_multipart_definition_is_logged_and_priced_by_model(self): + handler_result = self._run_handler(_make_multipart_response(FAST_URL, _definition_with_model(MAI_MODEL))) + + assert handler_result["kwargs"]["model"] == f"azure_speech/{MAI_MODEL}" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * MAI_PRICE_PER_SECOND) + assert handler_result["kwargs"]["standard_logging_object"]["model"] == f"azure_speech/{MAI_MODEL}" + + def test_fast_transcription_with_enhanced_mode_model_in_request_body_definition_is_priced_by_model(self): + handler_result = self._run_handler( + _make_response(FAST_URL), request_body={"definition": _definition_with_model(MAI_MODEL)} + ) + + assert handler_result["kwargs"]["model"] == f"azure_speech/{MAI_MODEL}" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * MAI_PRICE_PER_SECOND) + + def test_fast_transcription_with_enhanced_model_missing_from_cost_map_falls_back_to_standard_speech_price(self): + handler_result = self._run_handler( + _make_multipart_response(FAST_URL, _definition_with_model("MAI-Transcribe-1.5")) + ) + + assert handler_result["kwargs"]["model"] == "azure_speech/MAI-Transcribe-1.5" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND) + + def test_fast_transcription_with_enhanced_mode_disabled_keeps_standard_model_and_price(self): + handler_result = self._run_handler( + _make_multipart_response(FAST_URL, _definition_with_model(MAI_MODEL, enabled=False)) + ) + + assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND) + + def test_fast_transcription_without_enhanced_mode_keeps_standard_model_and_price(self): + handler_result = self._run_handler(_make_multipart_response(FAST_URL, json.dumps({"locales": ["en-US"]}))) + + assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND) + + def test_fast_transcription_with_malformed_definition_keeps_standard_model_and_price(self): + handler_result = self._run_handler(_make_multipart_response(FAST_URL, "not-json{")) + + assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND) + + class TestNormalizeDispatch: def test_normalize_routes_to_azure_speech_handler(self): normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(