fix(cost): price each realtime transcription clip at the model set when it ran

A session that changes its transcription model mid-way kept every logged clip
on the newest model. Each clip now takes the transcription model from the last
session event before it, and falls back to the session's own model
This commit is contained in:
mateo-berri 2026-10-31 04:17:49 -07:00
parent 0cdc1ccab7
commit 09cf450426
2 changed files with 42 additions and 30 deletions

View file

@ -6,6 +6,7 @@ import time
from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@ -3762,23 +3763,27 @@ def handle_realtime_transcription_cost_calculation(
- {"type": "duration", "seconds": <float>} → priced via input_cost_per_second
- {"type": "tokens", "input_tokens": ...} → priced via input/audio token cost
"""
completed_events: Final = [
cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
]
if not completed_events:
configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
clips: Final = tuple(
(as_str_mapping(result.get("usage")) or {}, configured)
for result, configured in zip(results, configured_models, strict=True)
if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
)
if not clips:
return 0.0
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
model_infos: Final = MappingProxyType(
{model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
)
override_info: Final = (
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
)
total_cost = 0.0
for event in completed_events:
usage = event.get("usage") or {}
total_cost += _transcription_usage_cost(usage, model_info, override_info)
return total_cost
return sum(
(_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
0.0,
)
_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
@ -3797,20 +3802,25 @@ def _configured_transcription_model(session: Mapping[str, object]) -> str | None
return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
def _session_transcription_model(result: Mapping[str, object]) -> str | None:
session: Final = _session_of(result)
return None if session is None else _configured_transcription_model(session)
def _newer_model(current: str | None, newer: str | None) -> str | None:
return newer or current
def _get_transcription_model_name_from_results(
results: OpenAIRealtimeStreamList,
) -> str | None:
sessions: Final = tuple(
session
for result in results
if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES
and (session := as_str_mapping(result.get("session"))) is not None
)
configured: Final = next(
(model for session in reversed(sessions) if (model := _configured_transcription_model(session))), None
)
return configured or next(
(session_model for session in sessions if (session_model := _str_field(session, "model"))), None
return next(
(model for result in results if (model := _str_field(_session_of(result), "model"))),
None,
)

View file

@ -1451,34 +1451,36 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type():
assert _transcription_usage_cost({}, {}) == 0.0
def test_realtime_transcription_prices_a_clip_at_the_latest_configured_transcription_model(monkeypatch):
def test_realtime_transcription_prices_each_clip_at_the_transcription_model_set_when_it_ran(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
from litellm.cost_calculator import handle_realtime_transcription_cost_calculation
realtime_model = "gpt-realtime-2.1-mini"
first_asr, latest_asr = "gpt-4o-transcribe", "gpt-transcribe"
first_asr, second_asr = "gpt-4o-transcribe", "gpt-transcribe"
first_rate = litellm.model_cost[first_asr]["input_cost_per_second"]
latest_rate = litellm.model_cost[latest_asr]["input_cost_per_second"]
assert first_rate != latest_rate and latest_rate > 0
second_rate = litellm.model_cost[second_asr]["input_cost_per_second"]
assert first_rate != second_rate and first_rate > 0 and second_rate > 0
def session_frame(frame_type: str, asr_model: str | None) -> dict[str, object]:
transcription = None if asr_model is None else {"model": asr_model}
return {"type": frame_type, "session": {"model": realtime_model, "audio": {"input": {"transcription": transcription}}}}
clip = {"type": "conversation.item.input_audio_transcription.completed", "usage": {"type": "duration", "seconds": 5}}
results: OpenAIRealtimeStreamList = [
session_frame("session.created", None),
session_frame("session.updated", first_asr),
session_frame("session.updated", latest_asr),
{"type": "conversation.item.input_audio_transcription.completed", "usage": {"type": "duration", "seconds": 5}},
clip,
session_frame("session.updated", second_asr),
clip,
]
cost = handle_realtime_transcription_cost_calculation(
results=results, custom_llm_provider="openai", litellm_model_name=realtime_model
)
assert cost == pytest.approx(5 * latest_rate, rel=1e-9)
assert cost == pytest.approx(5 * first_rate + 5 * second_rate, rel=1e-9)
def test_get_transcription_model_falls_back_to_session_model(monkeypatch):