scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge 5cdaa909b4 into 08f2a98250

Browse source
This commit is contained in:
meda anilkumar 2026-11-06 16:40:45 +09:00 • committed by GitHub
commit c5a32da124
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 201 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/model_prices_and_context_window_backup.json
View file

@ -12084,6 +12084,17 @@
"mode": "audio_speech",
"source": "https://azure.microsoft.com/en-us/pricing/calculator/"
},
"azure/speech/MAI-Transcribe-2": {
"audio_transcription_config": "azure_speech",
"input_cost_per_second": 0.00015,
"litellm_provider": "azure",
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/audio/transcriptions"
]
},
"azure/speech/azure-stt": {
"audio_transcription_config": "azure_speech",
"input_cost_per_second": 0.0002777778,

83
litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py
View file

@ -1,3 +1,5 @@
import json
import re
from collections.abc import Mapping, Sequence
from datetime import datetime
from typing import Final
@ -46,6 +48,45 @@ class AzureSpeechPassthroughLoggingHandler:
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_FAST_TRANSCRIPTION_MODEL}"
return f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{AZURE_SPEECH_BATCH_MODEL}"
@staticmethod
def _definition_from_multipart_body(httpx_response: httpx.Response) -> str | None:
content_type: Final = httpx_response.request.headers.get("content-type", "")
boundary_match: Final = re.search(r"boundary=(?P<boundary>[^;]+)", content_type)
if boundary_match is None:
return None
boundary: Final = boundary_match.group("boundary").strip().strip('"')
try:
request_content: Final = httpx_response.request.content
except RuntimeError:
return None
definition_pattern: Final = re.compile(
rb'Content-Disposition: form-data; name="definition"\r\n\r\n(?P<definition>.*?)\r\n--'
+ re.escape(boundary.encode()),
re.DOTALL,
)
definition_match: Final = definition_pattern.search(request_content)
if definition_match is None:
return None
return definition_match.group("definition").decode("utf-8", errors="ignore")
@staticmethod
def _enhanced_mode_model(request_body: Mapping[str, object], httpx_response: httpx.Response) -> str | None:
definition: object = request_body.get("definition")
if definition is None:
definition = AzureSpeechPassthroughLoggingHandler._definition_from_multipart_body(httpx_response)
if isinstance(definition, str):
try:
definition = json.loads(definition)
except json.JSONDecodeError:
return None
if not isinstance(definition, Mapping):
return None
enhanced_mode: Final = definition.get("enhancedMode")
if not isinstance(enhanced_mode, Mapping) or enhanced_mode.get("enabled") is not True:
return None
model: Final = enhanced_mode.get("model")
return model if isinstance(model, str) and model else None
@staticmethod
def _recognized_audio_seconds(response_body: Mapping[str, object] | Sequence[object] | None) -> float:
if not isinstance(response_body, Mapping):
@ -94,29 +135,48 @@ class AzureSpeechPassthroughLoggingHandler:
return AzureSpeechPassthroughLoggingHandler._fast_transcription_audio_seconds(response_body)
return 0.0
@staticmethod
def _price_with_model(pricing_model: str, audio_seconds: float) -> float:
prompt_cost, completion_cost = transcription_cost(
model=pricing_model,
custom_llm_provider="azure",
duration=audio_seconds,
)
return prompt_cost + completion_cost
@staticmethod
def _response_cost(
url_route: str,
httpx_response: httpx.Response,
response_body: Mapping[str, object] | Sequence[object] | None,
enhanced_mode_model: str | None = None,
) -> float:
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(
url_route, httpx_response, response_body
)
if audio_seconds <= 0.0:
return 0.0
pricing_model: Final = (
f"azure/speech/{enhanced_mode_model}" if enhanced_mode_model else AZURE_SPEECH_PRICING_MODEL
)
try:
prompt_cost, completion_cost = transcription_cost(
model=AZURE_SPEECH_PRICING_MODEL,
custom_llm_provider="azure",
duration=audio_seconds,
return AzureSpeechPassthroughLoggingHandler._price_with_model(pricing_model, audio_seconds)
except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row
if pricing_model == AZURE_SPEECH_PRICING_MODEL:
verbose_proxy_logger.warning(
"No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e
)
return 0.0
verbose_proxy_logger.warning(
"No price for %s, falling back to %s: %s", pricing_model, AZURE_SPEECH_PRICING_MODEL, e
)
try:
return AzureSpeechPassthroughLoggingHandler._price_with_model(AZURE_SPEECH_PRICING_MODEL, audio_seconds)
except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row
verbose_proxy_logger.warning(
"No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e
)
return 0.0
return prompt_cost + completion_cost
@staticmethod
def azure_speech_passthrough_handler(
@ -132,9 +192,18 @@ class AzureSpeechPassthroughLoggingHandler:
**kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler
) -> PassThroughEndpointLoggingTypedDict:
try:
model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route)
enhanced_mode_model: Final = (
AzureSpeechPassthroughLoggingHandler._enhanced_mode_model(request_body, httpx_response)
if AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route)
else None
)
model_name: Final = (
f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{enhanced_mode_model}"
if enhanced_mode_model
else AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route)
)
response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(
url_route, httpx_response, response_body
url_route, httpx_response, response_body, enhanced_mode_model
)
updated_kwargs: Final = {

11
model_prices_and_context_window.json
View file

@ -12084,6 +12084,17 @@
"mode": "audio_speech",
"source": "https://azure.microsoft.com/en-us/pricing/calculator/"
},
"azure/speech/MAI-Transcribe-2": {
"audio_transcription_config": "azure_speech",
"input_cost_per_second": 0.00015,
"litellm_provider": "azure",
"mode": "audio_transcription",
"output_cost_per_second": 0.0,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/audio/transcriptions"
]
},
"azure/speech/azure-stt": {
"audio_transcription_config": "azure_speech",
"input_cost_per_second": 0.0002777778,

103
tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py
View file

@ -279,6 +279,109 @@ class TestIsAzureSpeechRoute:
assert "response_cost" not in normalized["kwargs"]
MAI_MODEL = "MAI-Transcribe-2"
MAI_PRICE_PER_SECOND = 0.25
MULTIPART_BOUNDARY = "litellm-test-boundary"
def _make_multipart_response(url: str, definition: str | None) -> httpx.Response:
parts: Final = (
f"--{MULTIPART_BOUNDARY}\r\n"
'Content-Disposition: form-data; name="audio"; filename="audio.wav"\r\n'
"Content-Type: audio/wav\r\n\r\n"
)
definition_part: Final = (
f'--{MULTIPART_BOUNDARY}\r\nContent-Disposition: form-data; name="definition"\r\n\r\n{definition}\r\n'
if definition is not None
else ""
)
body: Final = f"{parts}FAKE-AUDIO\r\n{definition_part}--{MULTIPART_BOUNDARY}--\r\n".encode()
request = httpx.Request(
"POST",
url,
headers={"Content-Type": f"multipart/form-data; boundary={MULTIPART_BOUNDARY}"},
content=body,
)
return httpx.Response(200, request=request, text=json.dumps(FAST_BODY))
def _definition_with_model(model: str, enabled: bool = True) -> str:
return json.dumps({"enhancedMode": {"enabled": enabled, "model": model}})
class TestAzureSpeechEnhancedModePricing:
@pytest.fixture(autouse=True)
def mai_price(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setitem(
litellm.model_cost,
f"azure/speech/{MAI_MODEL}",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": MAI_PRICE_PER_SECOND,
"output_cost_per_second": 0.0,
},
)
def _run_handler(
self, httpx_response: httpx.Response, request_body: dict[str, object] | None = None
) -> dict[str, object]:
return AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=httpx_response,
response_body=FAST_BODY,
logging_obj=_make_logging_obj(),
url_route=FAST_URL,
result=json.dumps(FAST_BODY),
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body=request_body or {},
)
def test_fast_transcription_with_enhanced_mode_model_in_multipart_definition_is_logged_and_priced_by_model(self):
handler_result = self._run_handler(_make_multipart_response(FAST_URL, _definition_with_model(MAI_MODEL)))
assert handler_result["kwargs"]["model"] == f"azure_speech/{MAI_MODEL}"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * MAI_PRICE_PER_SECOND)
assert handler_result["kwargs"]["standard_logging_object"]["model"] == f"azure_speech/{MAI_MODEL}"
def test_fast_transcription_with_enhanced_mode_model_in_request_body_definition_is_priced_by_model(self):
handler_result = self._run_handler(
_make_response(FAST_URL), request_body={"definition": _definition_with_model(MAI_MODEL)}
)
assert handler_result["kwargs"]["model"] == f"azure_speech/{MAI_MODEL}"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * MAI_PRICE_PER_SECOND)
def test_fast_transcription_with_enhanced_model_missing_from_cost_map_falls_back_to_standard_speech_price(self):
handler_result = self._run_handler(
_make_multipart_response(FAST_URL, _definition_with_model("MAI-Transcribe-1.5"))
)
assert handler_result["kwargs"]["model"] == "azure_speech/MAI-Transcribe-1.5"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND)
def test_fast_transcription_with_enhanced_mode_disabled_keeps_standard_model_and_price(self):
handler_result = self._run_handler(
_make_multipart_response(FAST_URL, _definition_with_model(MAI_MODEL, enabled=False))
)
assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND)
def test_fast_transcription_without_enhanced_mode_keeps_standard_model_and_price(self):
handler_result = self._run_handler(_make_multipart_response(FAST_URL, json.dumps({"locales": ["en-US"]})))
assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND)
def test_fast_transcription_with_malformed_definition_keeps_standard_model_and_price(self):
handler_result = self._run_handler(_make_multipart_response(FAST_URL, "not-json{"))
assert handler_result["kwargs"]["model"] == "azure_speech/fast-transcription"
assert handler_result["kwargs"]["response_cost"] == pytest.approx(FAST_AUDIO_SECONDS * PRICE_PER_SECOND)
class TestNormalizeDispatch:
def test_normalize_routes_to_azure_speech_handler(self):
normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(