scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-04 07:27:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 195

Merge 3bb42da063 into e4832c5d06

Browse source
This commit is contained in:
ishaan-j-berri 2026-10-27 19:17:26 -07:00 • committed by GitHub
commit be3fb93cd8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 186 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/model_prices_and_context_window_backup.json
View file

@ -11241,6 +11241,20 @@
"/v1/audio/transcriptions"
]
},
"azure/speech/MAI-Transcribe-2": {
"input_cost_per_second": 0.00002777777777777778,
"output_cost_per_second": 0.0,
"litellm_provider": "azure",
"mode": "audio_transcription",
"source": "https://labs.ai.azure.com/innovations/mai-transcribe-2/"
},
"azure/speech/MAI-Transcribe-1.5": {
"input_cost_per_second": 0.0001,
"output_cost_per_second": 0.0,
"litellm_provider": "azure",
"mode": "audio_transcription",
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632"
},
"azure/tts-1": {
"deprecation_date": "2026-12-15",
"input_cost_per_character": 1.5e-05,

8
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -1787,6 +1787,14 @@ async def azure_speech_proxy_route(
)
raw_body: Final = await request.body()
if normalized_endpoint_path.endswith(
AZURE_SPEECH_FAST_TRANSCRIPTION_PATH
) and HttpPassThroughEndpointHelpers.is_multipart(request):
async with request.form() as form:
definition: Final = form.get("definition")
if isinstance(definition, str):
setattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, {"definition": definition})
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=str(target_url),

52
litellm/proxy/pass_through_endpoints/llm_provider_handlers/azure_speech_passthrough_logging_handler.py
View file

@ -4,6 +4,7 @@ from typing import Final
from urllib.parse import urlparse
import httpx
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
@ -26,9 +27,49 @@ from litellm.litellm_core_utils.litellm_logging import (
)
from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
from litellm.types.utils import StandardPassThroughResponseObject
from litellm.utils import get_model_cost_key
class _EnhancedMode(BaseModel):
model_config = ConfigDict(frozen=True, strict=True)
enabled: bool = False
model: str | None = Field(default=None, min_length=1)
class _FastTranscriptionDefinition(BaseModel):
model_config = ConfigDict(frozen=True, strict=True)
enhanced_mode: _EnhancedMode | None = Field(default=None, alias="enhancedMode")
class AzureSpeechPassthroughLoggingHandler:
@staticmethod
def _enhanced_transcription_model(url_route: str, request_body: Mapping[str, object]) -> str | None:
if not AzureSpeechPassthroughLoggingHandler._is_fast_transcription_route(url_route):
return None
definition: Final = request_body.get("definition")
if not isinstance(definition, str):
return None
try:
parsed: Final = _FastTranscriptionDefinition.model_validate_json(definition)
except ValidationError:
return None
if parsed.enhanced_mode is None or not parsed.enhanced_mode.enabled:
return None
return parsed.enhanced_mode.model
@staticmethod
def _resolve_model_and_pricing_model(url_route: str, request_body: Mapping[str, object]) -> tuple[str, str]:
enhanced_model: Final = AzureSpeechPassthroughLoggingHandler._enhanced_transcription_model(
url_route, request_body
)
if enhanced_model is None:
return AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route), AZURE_SPEECH_PRICING_MODEL
model_name: Final = f"{AZURE_SPEECH_CUSTOM_LLM_PROVIDER}/{enhanced_model}"
pricing_model: Final = get_model_cost_key(f"azure/speech/{enhanced_model}")
return model_name, pricing_model or AZURE_SPEECH_PRICING_MODEL
@staticmethod
def _is_short_audio_route(url_route: str) -> bool:
path: Final = urlparse(url_route).path
@ -99,6 +140,7 @@ class AzureSpeechPassthroughLoggingHandler:
url_route: str,
httpx_response: httpx.Response,
response_body: Mapping[str, object] | Sequence[object] | None,
pricing_model: str = AZURE_SPEECH_PRICING_MODEL,
) -> float:
audio_seconds: Final = AzureSpeechPassthroughLoggingHandler._billed_audio_seconds(
url_route, httpx_response, response_body
@ -107,13 +149,13 @@ class AzureSpeechPassthroughLoggingHandler:
return 0.0
try:
prompt_cost, completion_cost = transcription_cost(
model=AZURE_SPEECH_PRICING_MODEL,
model=pricing_model,
custom_llm_provider="azure",
duration=audio_seconds,
)
except Exception as e: # noqa: BLE001 # a missing price entry must not drop the spend log row
verbose_proxy_logger.warning(
"No price for %s, logging Azure Speech call at zero cost: %s", AZURE_SPEECH_PRICING_MODEL, e
"No price for %s, logging Azure Speech call at zero cost: %s", pricing_model, e
)
return 0.0
return prompt_cost + completion_cost
@ -132,9 +174,11 @@ class AzureSpeechPassthroughLoggingHandler:
**kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler
) -> PassThroughEndpointLoggingTypedDict:
try:
model_name: Final = AzureSpeechPassthroughLoggingHandler._model_from_url_route(url_route)
model_name, pricing_model = AzureSpeechPassthroughLoggingHandler._resolve_model_and_pricing_model(
url_route, request_body
)
response_cost: Final = AzureSpeechPassthroughLoggingHandler._response_cost(
url_route, httpx_response, response_body
url_route, httpx_response, response_body, pricing_model
)
updated_kwargs: Final = {

14
model_prices_and_context_window.json
View file

@ -11241,6 +11241,20 @@
"/v1/audio/transcriptions"
]
},
"azure/speech/MAI-Transcribe-2": {
"input_cost_per_second": 0.00002777777777777778,
"output_cost_per_second": 0.0,
"litellm_provider": "azure",
"mode": "audio_transcription",
"source": "https://labs.ai.azure.com/innovations/mai-transcribe-2/"
},
"azure/speech/MAI-Transcribe-1.5": {
"input_cost_per_second": 0.0001,
"output_cost_per_second": 0.0,
"litellm_provider": "azure",
"mode": "audio_transcription",
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/new-mai-models-in-microsoft-foundry-across-text-image-voice-and-speech/4524632"
},
"azure/tts-1": {
"deprecation_date": "2026-12-15",
"input_cost_per_character": 1.5e-05,

60
tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_azure_speech_passthrough_logging_handler.py
View file

@ -85,6 +85,66 @@ def _make_logging_obj() -> MagicMock:
class TestAzureSpeechPassthroughHandler:
@pytest.mark.parametrize(
"definition,expected_model,model_rate",
[
('{"enhancedMode":{"enabled":true,"model":"MAI-Transcribe-2"}}', "MAI-Transcribe-2", 0.01),
('{"enhancedMode":{"enabled":true,"model":"MAI-Transcribe-1.5"}}', "MAI-Transcribe-1.5", 0.02),
('{"enhancedMode":{"enabled":true,"model":"future-model"}}', "future-model", None),
('{"enhancedMode":{"enabled":false,"model":"MAI-Transcribe-2"}}', None, None),
('{"enhancedMode":{"model":"MAI-Transcribe-2"}}', None, None),
('{"enhancedMode":{"enabled":true}}', None, None),
('{"enhancedMode":{"enabled":true,"model":""}}', None, None),
('{"enhancedMode":{"enabled":"false","model":"MAI-Transcribe-2"}}', None, None),
('{"enhancedMode":[]}', None, None),
('{"enhancedMode":null}', None, None),
('{"enhancedMode":{"enabled":true,"model":123}}', None, None),
('{"locales":["en-US"]}', None, None),
("not json", None, None),
("[]", None, None),
(None, None, None),
],
)
def test_fast_transcription_logs_selected_model_and_uses_its_price_or_standard_fallback(
self,
monkeypatch: pytest.MonkeyPatch,
definition: str | None,
expected_model: str | None,
model_rate: float | None,
) -> None:
if model_rate is not None:
monkeypatch.setitem(
litellm.model_cost,
f"azure/speech/{expected_model}",
{
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": model_rate,
"output_cost_per_second": 0.0,
},
)
logging_obj: Final = _make_logging_obj()
handler_result: Final = AzureSpeechPassthroughLoggingHandler.azure_speech_passthrough_handler(
httpx_response=_make_response(FAST_URL),
response_body=FAST_BODY,
logging_obj=logging_obj,
url_route=FAST_URL,
result=json.dumps(FAST_BODY),
start_time=datetime(2026, 10, 26),
end_time=datetime(2026, 10, 26),
cache_hit=False,
request_body={"definition": definition},
)
model: Final = f"azure_speech/{expected_model or 'fast-transcription'}"
cost: Final = FAST_AUDIO_SECONDS * (model_rate if model_rate is not None else PRICE_PER_SECOND)
assert handler_result["kwargs"]["model"] == model
assert handler_result["kwargs"]["response_cost"] == pytest.approx(cost)
assert handler_result["kwargs"]["standard_logging_object"]["model"] == model
assert handler_result["kwargs"]["standard_logging_object"]["response_cost"] == pytest.approx(cost)
assert logging_obj.model_call_details["model"] == model
assert logging_obj.model_call_details["response_cost"] == pytest.approx(cost)
@pytest.mark.parametrize(
"url_route,expected_model,expected_cost",
[

55
tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
View file

@ -7700,6 +7700,13 @@ class TestAzureSpeechProxyRoute:
def test_fast_transcription_multipart_upload_is_forwarded_byte_for_byte(
self, azure_speech_client: TestClient
) -> None:
upload: Final = httpx.Request(
"POST",
f"http://testserver/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
)
uploaded_body: Final = upload.read()
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 640, "combinedPhrases": []})
@ -7708,17 +7715,18 @@ class TestAzureSpeechProxyRoute:
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_NON_UTF8_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
headers={"Authorization": "Bearer sk-virtual"},
content=uploaded_body,
headers={
"Authorization": "Bearer sk-virtual",
"Content-Type": upload.headers["content-type"],
},
)
assert response.status_code == 200
sent = route.calls.last.request
assert sent.headers["content-type"].startswith("multipart/form-data; boundary=")
assert sent.headers["content-type"] == upload.headers["content-type"]
assert dict(sent.url.params) == {"api-version": "2024-11-15"}
assert AZURE_SPEECH_NON_UTF8_WAV_BYTES in sent.content
assert b'name="definition"' in sent.content
assert sent.content == uploaded_body
assert sent.headers["ocp-apim-subscription-key"] == "server-subscription-key"
assert "authorization" not in sent.headers
@ -7769,8 +7777,21 @@ class TestAzureSpeechProxyRoute:
("azure_speech/batch-transcription", "azure_speech", 0.0)
]
@pytest.mark.parametrize(
"enhanced_model,pricing_model",
[
(None, "azure/speech/azure-stt"),
("MAI-Transcribe-2", "azure/speech/MAI-Transcribe-2"),
("MAI-Transcribe-1.5", "azure/speech/MAI-Transcribe-1.5"),
("future-model", "azure/speech/azure-stt"),
],
)
def test_fast_transcription_spend_is_priced_from_duration_milliseconds(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch
self,
azure_speech_client: TestClient,
monkeypatch: pytest.MonkeyPatch,
enhanced_model: str | None,
pricing_model: str,
) -> None:
from litellm.integrations.custom_logger import CustomLogger
@ -7794,24 +7815,32 @@ class TestAzureSpeechProxyRoute:
"output_cost_per_second": 0.0,
},
)
expected_rate: Final = litellm.model_cost[pricing_model]["input_cost_per_second"]
definition: Final = json.dumps(
{"locales": ["en-US"], "enhancedMode": {"enabled": True, "model": enhanced_model}}
if enhanced_model is not None
else {"locales": ["en-US"]}
)
with respx.mock(assert_all_called=True) as upstream:
upstream.post(f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}").mock(
return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []})
)
route: Final = upstream.post(
f"https://eastus.api.cognitive.microsoft.com{AZURE_SPEECH_FAST_ENDPOINT}"
).mock(return_value=httpx.Response(200, json={"durationMilliseconds": 5061, "combinedPhrases": []}))
response = azure_speech_client.post(
f"/azure_speech{AZURE_SPEECH_FAST_ENDPOINT}",
params={"api-version": "2024-11-15"},
files={"audio": ("eagle.wav", AZURE_SPEECH_WAV_BYTES, "audio/wav")},
data={"definition": json.dumps({"locales": ["en-US"]})},
data={"definition": definition},
headers={"Authorization": "Bearer sk-virtual"},
)
assert response.status_code == 200
assert definition.encode() in route.calls.last.request.content
assert AZURE_SPEECH_WAV_BYTES in route.calls.last.request.content
assert [(p["model"], p["custom_llm_provider"]) for p in recorder.payloads] == [
("azure_speech/fast-transcription", "azure_speech")
(f"azure_speech/{enhanced_model or 'fast-transcription'}", "azure_speech")
]
assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * 0.25)
assert recorder.payloads[0]["response_cost"] == pytest.approx(5.061 * expected_rate)
def test_short_audio_spend_is_priced_from_the_recognized_duration(
self, azure_speech_client: TestClient, monkeypatch: pytest.MonkeyPatch