scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge remote-tracking branch 'origin/main' into litellm_jwt_agent_id_claim

Browse source
This commit is contained in:
yassin 2026-09-14 19:34:14 +00:00
commit 486b328d99
29 changed files with 1383 additions and 155 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
litellm/cost_calculator.py
View file

@ -97,6 +97,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
ImageGenerationRequestQuality,
@ -2381,6 +2382,46 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]
return [attr for attr in field_names if attr != "cache_creation_tokens"]
def _combine_cached_tokens_details(
current: CachedTokensDetails | None, new: CachedTokensDetails
) -> CachedTokensDetails:
def _sum_optional(current_value: int | None, new_value: int | None) -> int | None:
if current_value is None and new_value is None:
return None
return (current_value or 0) + (new_value or 0)
return CachedTokensDetails(
text_tokens=_sum_optional(current.text_tokens if current is not None else None, new.text_tokens),
audio_tokens=_sum_optional(current.audio_tokens if current is not None else None, new.audio_tokens),
image_tokens=_sum_optional(current.image_tokens if current is not None else None, new.image_tokens),
)
def _combine_prompt_tokens_details(
current: PromptTokensDetailsWrapper | None, new: PromptTokensDetailsWrapper
) -> PromptTokensDetailsWrapper:
base: Final = current if current is not None else PromptTokensDetailsWrapper()
base_values: Final = MappingProxyType(
{attr: getattr(base, attr) for attr in type(base).model_fields if hasattr(base, attr)}
)
summed: Final = MappingProxyType(
{
attr: (getattr(base, attr, 0) or 0) + (getattr(new, attr) or 0)
for attr in _summable_prompt_token_fields(new)
if hasattr(new, attr) and isinstance(getattr(new, attr) or 0, (int, float))
}
)
new_cached_tokens_details: Final = getattr(new, "cached_tokens_details", None)
cached_tokens_details: Final = (
_combine_cached_tokens_details(getattr(base, "cached_tokens_details", None), new_cached_tokens_details)
if isinstance(new_cached_tokens_details, CachedTokensDetails)
else getattr(base, "cached_tokens_details", None)
)
return PromptTokensDetailsWrapper(
**MappingProxyType({**base_values, **summed, "cached_tokens_details": cached_tokens_details})
)
class BaseTokenUsageProcessor:
@staticmethod
def combine_usage_objects(usage_objects: list[Usage]) -> Usage:
@ -2389,7 +2430,6 @@ class BaseTokenUsageProcessor:
"""
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
Usage,
)
@ -2408,27 +2448,10 @@ class BaseTokenUsageProcessor:
and isinstance(current_val, (int, float))
):
setattr(combined, attr, current_val + new_val)
# Handle nested prompt_tokens_details
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details:
combined.prompt_tokens_details = PromptTokensDetailsWrapper()
# Check what keys exist in the model's prompt_tokens_details
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
for attr in _summable_prompt_token_fields(usage.prompt_tokens_details):
if (
hasattr(usage.prompt_tokens_details, attr)
and not attr.startswith("_")
and not callable(_attribute_value(usage.prompt_tokens_details, attr))
):
current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0
new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0
if new_val is not None and isinstance(new_val, (int, float)):
setattr(
combined.prompt_tokens_details,
attr,
current_val + new_val,
)
combined.prompt_tokens_details = _combine_prompt_tokens_details(
getattr(combined, "prompt_tokens_details", None), usage.prompt_tokens_details
)
# Handle nested completion_tokens_details
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:

38
litellm/integrations/otel/mappers/openinference.py
View file

@ -12,6 +12,7 @@ from typing import Final
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import (
MAX_MESSAGE_ATTRS_PER_SPAN,
MAX_TOOL_DEFINITION_ATTRS_PER_SPAN,
collect,
drop_none,
@ -26,6 +27,8 @@ from litellm.integrations.otel.model.payloads import (
ToolDefinition,
)
_MAX_INDEXED_MESSAGES: Final = MAX_MESSAGE_ATTRS_PER_SPAN // 2
class OpenInferenceMapper:
"""Emits OpenInference attributes for LLM_CALL spans.
@ -84,27 +87,44 @@ class OpenInferenceMapper:
return {}
def _llm_call(self, data: LLMCallSpanData) -> AttributeMap:
outputs: Final = output_messages(data)
indexed_in, indexed_out = self._indexed_split(len(data.messages_in), len(outputs))
return {
**collect(self._LLM_CALL_ATTRS, data),
**collect(self._BLOB_ATTRS, data),
**self._messages("llm.input_messages", "input.value", data.messages_in),
**self._messages("llm.output_messages", "output.value", output_messages(data)),
**self._messages(
"llm.input_messages",
"input.value",
data.messages_in,
self._prompt_positions(len(data.messages_in), indexed_in),
),
**self._messages("llm.output_messages", "output.value", outputs, range(indexed_out)),
**self._tools(data),
}
@staticmethod
def _messages(prefix: str, value_key: str, messages: Sequence[object]) -> AttributeMap:
"""Per-message ``{prefix}.{idx}.message.*`` keys + the ``value_key`` blob."""
def _indexed_split(inputs: int, outputs: int) -> tuple[int, int]:
"""Prompt and response share one allowance; the response is reserved at least half of it."""
indexed_out: Final = min(outputs, max(_MAX_INDEXED_MESSAGES // 2, _MAX_INDEXED_MESSAGES - inputs))
return _MAX_INDEXED_MESSAGES - indexed_out, indexed_out
@staticmethod
def _prompt_positions(total: int, indexed: int) -> tuple[int, ...]:
"""Prompt messages that get per-index attributes: message 0 and the most recent turns."""
if total <= indexed:
return tuple(range(total))
return (0, *range(total - indexed + 1, total))
@staticmethod
def _messages(prefix: str, value_key: str, messages: Sequence[object], positions: Sequence[int]) -> AttributeMap:
"""``{prefix}.{idx}.message.*`` keys for the messages at ``positions`` + the ``value_key`` blob of all."""
parsed: Final = [(m.get("role") if isinstance(m, dict) else None, message_content(m)) for m in messages]
attrs: Final = drop_none(
{
key: value
for idx, (role, content) in enumerate(parsed)
for idx, (role, content) in ((idx, parsed[idx]) for idx in positions)
for key, value in (
(
f"{prefix}.{idx}.message.role",
role if isinstance(role, str) else None,
),
(f"{prefix}.{idx}.message.role", role if isinstance(role, str) else None),
(f"{prefix}.{idx}.message.content", content),
)
}

8
litellm/integrations/otel/mappers/utils.py
View file

@ -32,6 +32,14 @@ core telemetry no matter how many vocabularies are configured.
"""
MAX_MESSAGE_ATTRS_PER_SPAN: Final = DEFAULT_SPAN_ATTRIBUTE_LIMIT // 8
"""Span-wide ceiling on per-index chat message attributes, prompt and response together.
An eighth is the largest share that still fits beside the tool ceiling and the core
of every vocabulary at once. The complete conversation still rides the JSON blobs.
"""
def tool_attr_budget(vocabularies: int) -> int:
"""Split the span-wide tool-definition ceiling across active vocabularies."""
return MAX_TOOL_DEFINITION_ATTRS_PER_SPAN // max(vocabularies, 1)

8
litellm/litellm_core_utils/litellm_logging.py
View file

@ -556,7 +556,7 @@ class Logging(LiteLLMLoggingBaseClass):
# ids leaking into a different, later request on the same thread. Sync
# support is deferred to a follow-up PR with its own safe-restore
# mechanism; async calls (the proxy's only call path) are unaffected.
if supports_correlation_logging:
if supports_correlation_logging and litellm.request_correlation_in_logs:
set_trace_id(self.litellm_trace_id)
set_session_id(self.litellm_session_id)
# set_trace_id()/set_session_id() sanitize (strip control chars, bound
@ -2442,7 +2442,7 @@ class Logging(LiteLLMLoggingBaseClass):
call) would leave the outer request's subsequent log lines stamped with
the nested call's trace_id/session_id instead of its own.
Uses a plain set() of the captured pre-call value rather than
Uses a plain contextvar set() of the captured pre-call value rather than
contextvars.Token-based reset(), since this can end up called from a
different asyncio Task/context than __init__ ran in (e.g. the request
task's own wrapper() finally block, plus async_success_handler
@ -2453,8 +2453,8 @@ class Logging(LiteLLMLoggingBaseClass):
that Task's view of the contextvars, so calling it multiple times
(once per Task involved in this attempt) is required, not just safe.
"""
set_trace_id(self._pre_call_trace_id)
set_session_id(self._pre_call_session_id)
trace_id_var.set(self._pre_call_trace_id)
session_id_var.set(self._pre_call_session_id)
def _restore_correlation_context_if_unclaimed(self) -> None:
"""Guarded variant for __del__-triggered cleanup only.

80
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -9,6 +9,8 @@ from types import MappingProxyType
from typing import Any, Final, Literal, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
import litellm
from litellm._internal_context import current_billing_time
from litellm._logging import verbose_logger
@ -772,6 +774,7 @@ def calculate_cache_writing_cost(
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_hit_audio_tokens: ReadOnly[int]
cache_creation_tokens: int
cache_creation_token_details: CacheCreationTokenDetails | None
text_tokens: int
@ -802,12 +805,34 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
)
or None
)
text_tokens: Final = (
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None)
cached_audio_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens
)
cached_text_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "text_tokens") or 0,
cache_hit_tokens - cached_audio_tokens,
)
cached_image_tokens: Final = min(
_get_token_detail_value(cached_tokens_details, "image_tokens") or 0,
cache_hit_tokens - cached_audio_tokens - cached_text_tokens,
)
text_tokens: Final = max(
(
cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
)
- cached_text_tokens,
0,
)
audio_tokens: Final = max(
(cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens,
0,
)
image_tokens: Final = max(
(cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens,
0,
)
audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0
image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0
video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0))
character_count: Final = (
cast(
@ -835,6 +860,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_hit_audio_tokens=cached_audio_tokens,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
@ -918,7 +944,16 @@ def _calculate_input_cost(
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
audio_cache_read_rate: Final = _get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
prompt_cost += float(cache_hit_audio_tokens) * (
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"]:
@ -1149,6 +1184,7 @@ def generic_cost_per_token(
### PROCESSING COST
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
@ -1319,6 +1355,7 @@ class BilledTokenRates:
input_cost_per_token: float
output_cost_per_token: float
cache_read_input_token_cost: float
cache_read_input_audio_token_cost: float
cache_creation_input_token_cost: float
cache_creation_input_token_cost_above_1hr: float
output_cost_per_reasoning_token: float
@ -1330,6 +1367,7 @@ class BilledTokenRates:
input_cost_per_token=self.input_cost_per_token * multiplier,
output_cost_per_token=self.output_cost_per_token * multiplier,
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
@ -1353,15 +1391,16 @@ def _reasoning_token_count(usage: Usage) -> int:
return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]:
"""(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details
first, then the private top-level counters the Usage constructor mirrors cache tokens onto for
providers/callers that bypass the details."""
def _cache_token_counts(usage: Usage) -> tuple[int, int, int, CacheCreationTokenDetails | None]:
"""(cache read tokens, cached audio tokens, cache creation tokens, cache creation details): read from
prompt_tokens_details first, then the private top-level counters the Usage constructor mirrors cache
tokens onto for providers/callers that bypass the details."""
parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None
parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0
parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0
return (
parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)),
parsed["cache_hit_audio_tokens"] if parsed is not None else 0,
parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)),
parsed["cache_creation_token_details"] if parsed is not None else None,
)
@ -1372,11 +1411,13 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does."""
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
output_rate: Final = custom_cost_per_token["output_cost_per_token"]
cache_read_rate: Final = custom_cost_per_token.get("cache_read_input_token_cost", input_rate)
cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate)
return BilledTokenRates(
input_cost_per_token=input_rate,
output_cost_per_token=output_rate,
cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
cache_read_input_token_cost=cache_read_rate,
cache_read_input_audio_token_cost=cache_read_rate,
cache_creation_input_token_cost=cache_creation_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
output_cost_per_reasoning_token=output_rate,
@ -1413,6 +1454,11 @@ def _cost_map_billed_rates(
completion_base_cost=completion_base_cost,
current_time=billing_time,
)
audio_cache_read_rate: Final = _get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
multiplier: Final = (
_get_regional_uplift_multiplier(model_info, data_residency)
* get_vertex_regional_endpoint_uplift(model_info, vertex_location)
@ -1422,6 +1468,9 @@ def _cost_map_billed_rates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
cache_read_input_token_cost=cache_read_cost_rate,
cache_read_input_audio_token_cost=(
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate
),
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
output_cost_per_reasoning_token=reasoning_rate,
@ -1494,7 +1543,9 @@ def get_token_type_cost_breakdown(
if rates is None:
return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage)
cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
usage
)
cache_creation_cost: Final = (
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
if custom_cost_per_token is not None
@ -1507,7 +1558,10 @@ def get_token_type_cost_breakdown(
)
return TokenTypeCostBreakdown(
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost,
cache_read_cost=(
float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
+ float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
),
cache_creation_cost=cache_creation_cost,
rates=rates,
)

151
litellm/model_prices_and_context_window_backup.json
View file

@ -5513,7 +5513,8 @@
},
"azure/gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-03-02",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5546,7 +5547,8 @@
},
"azure/gpt-realtime-1.5-2026-02-23": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-08-24",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5683,6 +5685,7 @@
},
"azure/gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -5715,6 +5718,7 @@
},
"azure/gpt-realtime-mini-2025-10-06": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -7409,6 +7413,80 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-chat-latest": {
"cache_read_input_token_cost": 5e-07,
"deprecation_date": "2026-12-02",
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"reasoning_effort_levels": [
"medium"
],
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/chat-latest": {
"cache_read_input_token_cost": 5e-07,
"deprecation_date": "2026-12-02",
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"reasoning_effort_levels": [
"medium"
],
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/us/gpt-5.6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
@ -7675,6 +7753,43 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure/us/gpt-chat-latest": {
"cache_read_input_token_cost": 5.5e-07,
"deprecation_date": "2026-12-02",
"input_cost_per_token": 5.5e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.3e-05,
"reasoning_effort_levels": [
"medium"
],
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/eu/gpt-5.6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
@ -23053,34 +23168,6 @@
"output_cost_per_token": 0.0,
"source": "https://fireworks.ai/pricing"
},
"friendliai/meta-llama-3.1-70b-instruct": {
"input_cost_per_token": 6e-07,
"litellm_provider": "friendliai",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"friendliai/meta-llama-3.1-8b-instruct": {
"input_cost_per_token": 1e-07,
"litellm_provider": "friendliai",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"friendliai/zai-org/GLM-5.3-Flash": {
"litellm_provider": "friendliai",
"max_input_tokens": 1048576,
@ -32678,6 +32765,7 @@
},
"gpt-realtime": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,
@ -32711,6 +32799,7 @@
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -32847,6 +32936,7 @@
"gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 1e-05,
"input_cost_per_token": 6e-07,
@ -32878,6 +32968,7 @@
},
"gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,

30
litellm/proxy/_types.py
View file

@ -4838,6 +4838,14 @@ class JWTIssuerConfig(BaseModel):
default=None,
description="Issuer-specific claim path to normalize into LiteLLM's end-user id.",
)
virtual_key_claim_field: str | None = Field(
default=None,
description="Issuer-specific claim path used for the virtual key mapping lookup. Falls back to the global field.",
)
unregistered_jwt_client_behavior: UnregisteredJWTClientBehavior | None = Field(
default=None,
description="Issuer-specific policy when the virtual key claim has no mapping. Falls back to the global policy.",
)
model_config = {
"extra": "forbid",
@ -5072,6 +5080,28 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
super().__init__(**kwargs)
def get_issuer_config(self, issuer: str | None) -> JWTIssuerConfig | None:
if issuer is None or self.issuers is None:
return None
return next((config for config in self.issuers if config.issuer == issuer), None)
def is_virtual_key_mapping_configured(self) -> bool:
if self.virtual_key_claim_field is not None:
return True
return any(config.virtual_key_claim_field is not None for config in self.issuers or ())
def get_virtual_key_claim_field(self, issuer: str | None) -> str | None:
issuer_config: Final = self.get_issuer_config(issuer)
if issuer_config is not None and issuer_config.virtual_key_claim_field is not None:
return issuer_config.virtual_key_claim_field
return self.virtual_key_claim_field
def get_unregistered_jwt_client_behavior(self, issuer: str | None) -> UnregisteredJWTClientBehavior:
issuer_config: Final = self.get_issuer_config(issuer)
if issuer_config is not None and issuer_config.unregistered_jwt_client_behavior is not None:
return issuer_config.unregistered_jwt_client_behavior
return self.unregistered_jwt_client_behavior
class PrismaCompatibleUpdateDBModel(TypedDict, total=False):
model_name: str

19
litellm/proxy/auth/user_api_key_auth.py
View file

@ -996,9 +996,12 @@ async def _resolve_jwt_to_virtual_key(
- Raises HTTPException: REJECT policy hit, missing claim under
REJECT/AUTO_REGISTER, or other policy violations.
"""
virtual_key_claim_field: Final = jwt_handler.litellm_jwtauth.virtual_key_claim_field
raw_issuer: Final = jwt_claims.get(JWTHandler.LITELLM_JWT_ISSUER_CLAIM)
normalized_issuer: Final = raw_issuer if isinstance(raw_issuer, str) else None
virtual_key_claim_field: Final = jwt_handler.litellm_jwtauth.get_virtual_key_claim_field(normalized_issuer)
if virtual_key_claim_field is None:
return None
behavior: Final = jwt_handler.litellm_jwtauth.get_unregistered_jwt_client_behavior(normalized_issuer)
claim_value: Final = get_nested_value(
data=jwt_claims,
@ -1015,7 +1018,6 @@ async def _resolve_jwt_to_virtual_key(
# simply by presenting a JWT that omits the configured field. For
# AUTO_REGISTER there is no stable identity to map without a claim
# value, so we deny rather than create a sentinel-keyed record.
behavior = jwt_handler.litellm_jwtauth.unregistered_jwt_client_behavior
if behavior in (
UnregisteredJWTClientBehavior.REJECT,
UnregisteredJWTClientBehavior.AUTO_REGISTER,
@ -1030,7 +1032,13 @@ async def _resolve_jwt_to_virtual_key(
return None
cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key)
raw_cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key)
sentinel_written_by_this_policy: Final = behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER
cached_mapping: Final = (
None
if raw_cached_mapping == _JWT_PROXY_ADMIN_SENTINEL and not sentinel_written_by_this_policy
else raw_cached_mapping
)
if cached_mapping == _JWT_PROXY_ADMIN_SENTINEL:
# Previously resolved to a proxy admin via auth_builder; skip the
@ -1039,7 +1047,6 @@ async def _resolve_jwt_to_virtual_key(
return None
if cached_mapping == "__NO_MAPPING__":
behavior = jwt_handler.litellm_jwtauth.unregistered_jwt_client_behavior
if behavior == UnregisteredJWTClientBehavior.REJECT:
raise HTTPException(
status_code=403,
@ -1102,8 +1109,6 @@ async def _resolve_jwt_to_virtual_key(
)
# No mapping found (DB miss or no DB) — apply no-match policy.
behavior = jwt_handler.litellm_jwtauth.unregistered_jwt_client_behavior
if behavior == UnregisteredJWTClientBehavior.REJECT:
# Cache the miss before raising so repeated rejections are served from
# cache and don't re-query the DB on every request.
@ -1483,7 +1488,7 @@ async def _user_api_key_auth_builder(
# unnecessary DB queries in auth_builder
do_standard_jwt_auth = True
pending_auto_register: _PendingAutoRegister | None = None
if jwt_handler.litellm_jwtauth.virtual_key_claim_field is not None:
if jwt_handler.litellm_jwtauth.is_virtual_key_mapping_configured():
# Decode JWT to get claims without running full auth_builder
jwt_claims: dict | None
if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled and not is_jwt:

36
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -43,6 +43,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.responses.litellm_completion_transformation.session_handler import (
ResponsesSessionHandler,
)
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -2816,27 +2817,24 @@ class LiteLLMCompletionResponsesConfig:
# Translate prompt_tokens_details to input_tokens_details
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None:
prompt_details: Final = usage.prompt_tokens_details
input_details_dict: Final[dict[str, int]] = {}
if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None:
input_details_dict["cached_tokens"] = prompt_details.cached_tokens
else:
input_details_dict["cached_tokens"] = 0
if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None:
input_details_dict["text_tokens"] = prompt_details.text_tokens
if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None:
input_details_dict["audio_tokens"] = prompt_details.audio_tokens
cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr(
cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None)
cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr(
prompt_details, "cache_creation_tokens", None
)
if cache_write_tokens is not None:
input_details_dict["cache_write_tokens"] = cache_write_tokens
if input_details_dict:
response_usage.input_tokens_details = InputTokensDetails(**input_details_dict)
cache_write_extra: Final[Mapping[str, int]] = (
MappingProxyType({"cache_write_tokens": cache_write_tokens})
if cache_write_tokens is not None
else MappingProxyType({})
)
response_usage.input_tokens_details = InputTokensDetails(
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
text_tokens=prompt_details.text_tokens,
audio_tokens=prompt_details.audio_tokens,
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
)
# Translate completion_tokens_details to output_tokens_details
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None:

3
litellm/responses/utils.py
View file

@ -1179,6 +1179,9 @@ class ResponseAPILoggingUtils:
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
cached_tokens_details=getattr(
response_api_usage.input_tokens_details, "cached_tokens_details", None
),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
)
completion_tokens_details: CompletionTokensDetailsWrapper | None = None

6
litellm/types/llms/base.py
View file

@ -75,3 +75,9 @@ class HiddenParams(OpenAIObject):
data: Final = super().model_dump(**kwargs)
data["_response_ms"] = self._response_ms
return data
class CachedTokensDetails(BaseModel):
text_tokens: int | None = None
audio_tokens: int | None = None
image_tokens: int | None = None

10
litellm/types/llms/openai.py
View file

@ -91,6 +91,8 @@ from litellm.types.responses.main import (
OutputImageGenerationCall,
)
from .base import CachedTokensDetails
FileContent = IO[bytes] | bytes | PathLike
FileTypes = (
@ -1288,6 +1290,7 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
text_tokens: int | None = None
model_config = {"extra": "allow"}
@ -2254,10 +2257,17 @@ class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict):
usage: NotRequired[ReadOnly[Mapping[str, object]]]
class OpenAIRealtimeCachedTokensDetails(TypedDict, total=False):
text_tokens: ReadOnly[int]
audio_tokens: ReadOnly[int]
image_tokens: ReadOnly[int]
class OpenAIRealtimeUsageTokenDetails(TypedDict):
audio_tokens: ReadOnly[int]
text_tokens: ReadOnly[int]
cached_tokens: NotRequired[ReadOnly[int]]
cached_tokens_details: NotRequired[ReadOnly[OpenAIRealtimeCachedTokensDetails]]
class OpenAIRealtimeResponseUsage(TypedDict):

7
litellm/types/utils.py
View file

@ -48,6 +48,7 @@ from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.types.llms.base import (
BaseLiteLLMOpenAIResponseObject,
CachedTokensDetails,
LiteLLMPydanticObjectBase,
)
from litellm.types.mcp import MCPServerCostInfo
@ -252,6 +253,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
@ -1710,6 +1712,9 @@ class PromptTokensDetailsWrapper(
cache_creation_token_details: CacheCreationTokenDetails | None = None
"""Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching."""
cached_tokens_details: CachedTokensDetails | None = None
"""Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split."""
def __setattr__(self, name: str, value: object) -> None:
super().__setattr__(name, value)
if name == "cache_write_tokens":
@ -1756,6 +1761,8 @@ class PromptTokensDetailsWrapper(
del self.cache_creation_tokens
if self.cache_creation_token_details is None:
del self.cache_creation_token_details
if self.cached_tokens_details is None:
del self.cached_tokens_details
class ServerToolUse(BaseModel):

1
litellm/utils.py
View file

@ -5882,6 +5882,7 @@ def _get_model_info_helper(
"cache_creation_input_token_cost_ultrafast", None
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens", None

151
model_prices_and_context_window.json
View file

@ -5513,7 +5513,8 @@
},
"azure/gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-03-02",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5546,7 +5547,8 @@
},
"azure/gpt-realtime-1.5-2026-02-23": {
"cache_creation_input_audio_token_cost": 4e-06,
"cache_read_input_token_cost": 4e-06,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-08-24",
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -5683,6 +5685,7 @@
},
"azure/gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -5715,6 +5718,7 @@
},
"azure/gpt-realtime-mini-2025-10-06": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
@ -7409,6 +7413,80 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-chat-latest": {
"cache_read_input_token_cost": 5e-07,
"deprecation_date": "2026-12-02",
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"reasoning_effort_levels": [
"medium"
],
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/chat-latest": {
"cache_read_input_token_cost": 5e-07,
"deprecation_date": "2026-12-02",
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"reasoning_effort_levels": [
"medium"
],
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/us/gpt-5.6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
@ -7675,6 +7753,43 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure/us/gpt-chat-latest": {
"cache_read_input_token_cost": 5.5e-07,
"deprecation_date": "2026-12-02",
"input_cost_per_token": 5.5e-06,
"litellm_provider": "azure",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3.3e-05,
"reasoning_effort_levels": [
"medium"
],
"source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"azure/eu/gpt-5.6": {
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
@ -23053,34 +23168,6 @@
"output_cost_per_token": 0.0,
"source": "https://fireworks.ai/pricing"
},
"friendliai/meta-llama-3.1-70b-instruct": {
"input_cost_per_token": 6e-07,
"litellm_provider": "friendliai",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"friendliai/meta-llama-3.1-8b-instruct": {
"input_cost_per_token": 1e-07,
"litellm_provider": "friendliai",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"friendliai/zai-org/GLM-5.3-Flash": {
"litellm_provider": "friendliai",
"max_input_tokens": 1048576,
@ -32678,6 +32765,7 @@
},
"gpt-realtime": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,
@ -32711,6 +32799,7 @@
},
"gpt-realtime-1.5": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
@ -32847,6 +32936,7 @@
"gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 1e-05,
"input_cost_per_token": 6e-07,
@ -32878,6 +32968,7 @@
},
"gpt-realtime-2025-08-28": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-01-20",
"input_cost_per_audio_token": 3.2e-05,

4
osv-scanner.toml
View file

@ -5,5 +5,5 @@ reason = "diskcache has no fixed release published; remove this entry once one e
[[IgnoredVulns]]
id = "GHSA-h7x2-h6g9-p789"
ignoreUntil = 2026-09-14
reason = "mlflow has no fixed release published; remove this entry once one exists"
ignoreUntil = 2026-10-14
reason = "mlflow has no fixed release published (3.16.0, 2026-09-04, and master still store gateway secret api_base unvalidated); remove this entry once one exists"

11
tests/proxy_behavior/auth/test_auth_object_prefetch.py
View file

@ -22,6 +22,11 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
pytestmark = pytest.mark.asyncio(loop_scope="session")
def _frozen_cache() -> UserApiKeyCache:
"""The org entries carry a 5s TTL; a frozen clock keeps a slow first call from expiring them mid-test."""
return UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1_000_000.0), redis_cache=None)
def _dead_db() -> MagicMock:
prisma = MagicMock(name="prisma_client")
prisma.db.query_first = AsyncMock(return_value=None)
@ -58,7 +63,7 @@ async def test_join_binds_the_membership_to_the_requested_team(prisma):
data={"user_id": user_id, "team_id": team_b, "litellm_budget_table": {"connect": {"budget_id": f"b-{run}"}}}
)
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None)
cache = _frozen_cache()
refs = AuthObjectRefs(user_id=user_id, team_id=team_a, membership_user_id=user_id, organization_id=org_id)
await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma)
@ -100,7 +105,7 @@ async def test_join_reads_team_model_aliases_from_the_mapped_column(prisma):
where={"team_id": team_id}, include={"litellm_model_table": True}
)
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None)
cache = _frozen_cache()
refs = AuthObjectRefs(user_id=None, team_id=team_id, membership_user_id=None, organization_id=None)
await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma)
@ -144,7 +149,7 @@ async def test_join_reads_null_nested_lists_the_way_prisma_does(prisma):
where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}}, include={"litellm_budget_table": True}
)
cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None)
cache = _frozen_cache()
refs = AuthObjectRefs(user_id=user_id, team_id=team_id, membership_user_id=user_id, organization_id=None)
await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma)

161
tests/test_litellm/integrations/otel/test_otel_v2_emitter.py
View file

@ -1,6 +1,8 @@
"""Golden tests for the OTel v2 engine: span shape, kinds, semconv attributes,
legacy dual-emit, hierarchy, error status, and idempotency. Needs the OTel SDK."""
import json
import pytest
pytest.importorskip("opentelemetry")
@ -18,6 +20,7 @@ from litellm.integrations.otel.plumbing import providers # noqa: E402
from litellm.integrations.otel.emitter import SpanEmitter # noqa: E402
from litellm.integrations.otel.emitter import stamp_error # noqa: E402
from litellm.integrations.otel.mappers.utils import ( # noqa: E402
MAX_MESSAGE_ATTRS_PER_SPAN,
MAX_TOOL_DEFINITION_ATTRS_PER_SPAN,
)
from litellm.integrations.otel.model.payloads import ( # noqa: E402
@ -440,3 +443,161 @@ def test_vendor_tool_definitions_are_truncated_not_dropped():
assert a["llm.tools.0.tool.name"] == "tool_0"
assert a["llm.tools.0.tool.json_schema"]
assert "llm.tools.126.tool.name" not in a
def _conversation_payload(turns, choices=1, **overrides):
"""A ``turns``-message chat with ``choices`` response choices, content-bearing."""
return _payload(
messages=[{"role": ("user", "assistant")[i % 2], "content": f"turn {i}"} for i in range(turns)],
response={
"id": "resp_1",
"model": "gpt-4o-2024",
"choices": [
{"finish_reason": "stop", "message": {"role": "assistant", "content": f"reply {i}"}}
for i in range(choices)
],
},
**overrides,
)
def _conversation_span(mapper_names, payload, legacy_compat=False):
"""The exported LLM-call span for ``payload`` with content capture on."""
cfg = OpenTelemetryV2Config(
exporter="in_memory",
legacy_compat=legacy_compat,
mapper_names=list(mapper_names),
capture_message_content="span_only",
)
provider, exporter = providers.in_memory_provider(cfg)
engine = SpanEmitter(providers.get_tracer(provider, "litellm-test"), cfg)
engine.emit(
SpanRole.LLM_CALL,
LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True),
)
(span,) = exporter.get_finished_spans()
return span
def _indexed_message_count(attributes, prefix):
return len({key.split(".")[2] for key in attributes if key.startswith(f"{prefix}.")})
@pytest.mark.parametrize("turns", [60, 200])
def test_long_conversation_does_not_evict_core_attributes(turns):
"""Per-message OpenInference attributes must never crowd core telemetry off the span."""
span = _conversation_span(["genai", "openinference"], _conversation_payload(turns))
a = span.attributes
assert span.dropped_attributes == 0
assert a[GenAI.REQUEST_MODEL] == "gpt-4o"
assert a[GenAI.PROVIDER_NAME] == "openai"
assert a[GenAI.USAGE_INPUT_TOKENS] == 10
assert a[GenAI.USAGE_OUTPUT_TOKENS] == 5
assert a[GenAI.RESPONSE_FINISH_REASONS] == ("stop",)
assert a[f"{LiteLLM.COST_PREFIX}total"] == 0.002
assert a["llm.input_messages.0.message.content"] == "turn 0"
assert a["llm.output_messages.0.message.content"] == "reply 0"
assert a[f"llm.input_messages.{turns - 1}.message.content"] == f"turn {turns - 1}"
assert f"llm.input_messages.{turns // 2}.message.role" not in a
assert len(json.loads(a["input.value"])) == turns
assert len(json.loads(a["output.value"])) == 1
assert len(json.loads(a[GenAI.INPUT_MESSAGES])) == turns
def test_short_conversation_keeps_every_message_indexed():
"""Below the cap nothing is truncated in either direction."""
a = _conversation_span(["genai", "openinference"], _conversation_payload(4, choices=2)).attributes
for idx in range(4):
assert a[f"llm.input_messages.{idx}.message.content"] == f"turn {idx}"
for idx in range(2):
assert a[f"llm.output_messages.{idx}.message.content"] == f"reply {idx}"
def test_indexed_prompt_keeps_opener_and_latest_turns_under_a_value_length_limit(monkeypatch):
"""The system prompt and the live turn keep their own keys once the SDK clips ``input.value``."""
monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", "256")
chat = _conversation_payload(60)
payload = {
**chat,
"messages": [
{"role": "system", "content": "be terse"},
*chat["messages"][1:-1],
{"role": "user", "content": "LATEST-TURN"},
],
}
a = _conversation_span(["genai", "openinference"], payload).attributes
assert len(a["input.value"]) == 256
assert a["llm.input_messages.0.message.role"] == "system"
assert a["llm.input_messages.0.message.content"] == "be terse"
assert a["llm.input_messages.59.message.role"] == "user"
assert a["llm.input_messages.59.message.content"] == "LATEST-TURN"
assert a["llm.output_messages.0.message.content"] == "reply 0"
assert [int(key.split(".")[2]) for key in a if key.endswith("message.content") and key.startswith("llm.input_")] == [
0,
*range(54, 60),
]
def test_message_cap_is_shared_across_input_and_output():
"""One span-wide allowance covers both directions, and the response always keeps a share."""
long_prompt = _conversation_span(["genai", "openinference"], _conversation_payload(60, choices=1)).attributes
many_choices = _conversation_span(["genai", "openinference"], _conversation_payload(60, choices=20)).attributes
single_reply_indexed = _indexed_message_count(long_prompt, "llm.output_messages")
assert single_reply_indexed == 1
assert _indexed_message_count(long_prompt, "llm.input_messages") + single_reply_indexed == (
MAX_MESSAGE_ATTRS_PER_SPAN // 2
)
assert _indexed_message_count(many_choices, "llm.input_messages") > 0
assert _indexed_message_count(many_choices, "llm.output_messages") > single_reply_indexed
assert _indexed_message_count(many_choices, "llm.input_messages") + _indexed_message_count(
many_choices, "llm.output_messages"
) == (MAX_MESSAGE_ATTRS_PER_SPAN // 2)
def test_fully_populated_span_with_every_vocabulary_stays_within_the_attribute_limit():
"""Every capped family maxed at once still leaves the whole core intact."""
payload = _conversation_payload(
200,
choices=20,
stream=True,
model_parameters={
**_tools_payload(127)["model_parameters"],
"top_p": 0.9,
"frequency_penalty": 0.1,
"presence_penalty": 0.1,
"seed": 7,
"stop": ["\n"],
},
cost_breakdown={
key: 0.001
for key in (
"input_cost",
"output_cost",
"cache_read_cost",
"cache_creation_cost",
"tool_usage_cost",
"original_cost",
"discount_amount",
"discount_percent",
"margin_fixed_amount",
"margin_percent",
"margin_total_amount",
"total_cost",
)
},
)
span = _conversation_span(["genai", "openinference", "langfuse", "weave", "langtrace"], payload, legacy_compat=True)
a = span.attributes
assert span.dropped_attributes == 0
assert a[GenAI.REQUEST_MODEL] == "gpt-4o"
assert a[f"{LiteLLM.COST_PREFIX}total"] == 0.002
assert a[LiteLLM.TOOLS_DECLARED] == 127
assert a["llm.input_messages.0.message.content"] == "turn 0"
assert a["llm.input_messages.199.message.content"] == "turn 199"
assert a["llm.output_messages.0.message.content"] == "reply 0"

187
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -2101,6 +2101,37 @@ def test_generic_cost_per_token_azure_gpt_6_astra_foundry_price_sheet(
assert completion_cost == pytest.approx(zone_multiplier * output_multiplier * completion_tokens * 5e-5)
@pytest.mark.parametrize(
"model,input_rate,cache_read_rate,output_rate",
[
("azure/gpt-chat-latest", 5e-6, 5e-7, 3e-5),
("azure/chat-latest", 5e-6, 5e-7, 3e-5),
("azure/us/gpt-chat-latest", 5.5e-6, 5.5e-7, 3.3e-5),
],
)
def test_generic_cost_per_token_azure_gpt_chat_latest_price_sheet(
_local_model_cost_map, model, input_rate, cache_read_rate, output_rate
):
"""The Azure OpenAI price sheet lists GPT-Chat Latest at $5 input, $0.50 cached input and $30 output per 1M
tokens on Global, and $5.50, $0.55 and $33 on Data Zone. Foundry names the product gpt-chat-latest and the
OpenAI API names the same model chat-latest, so both spellings bill the Global sheet.
"""
prompt_tokens = 100000
cached_tokens = 40000
completion_tokens = 1000
usage = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
)
prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="azure")
assert prompt_cost == pytest.approx((prompt_tokens - cached_tokens) * input_rate + cached_tokens * cache_read_rate)
assert completion_cost == pytest.approx(completion_tokens * output_rate)
def test_generic_cost_per_token_azure_ai_gpt_6_astra_flex_bills_the_standard_rate(_local_model_cost_map):
usage = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100)
@ -2648,6 +2679,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
prompt_tokens_details: PromptTokensDetailsResult = {
"cache_hit_tokens": 0,
"cache_hit_audio_tokens": 0,
"cache_creation_tokens": 0,
"cache_creation_token_details": CacheCreationTokenDetails(
ephemeral_5m_input_tokens=100,
@ -4005,6 +4037,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp
input_cost_per_token=6e-6,
output_cost_per_token=3e-5,
cache_read_input_token_cost=6e-7,
cache_read_input_audio_token_cost=6e-7,
cache_creation_input_token_cost=7.5e-6,
cache_creation_input_token_cost_above_1hr=0.0,
output_cost_per_reasoning_token=3e-5,
@ -5147,3 +5180,157 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output(
assert completion_cost == pytest.approx(
30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"]
)
def test_cached_realtime_audio_tokens_billed_at_audio_cache_read_rate(
_local_model_cost_map: None,
) -> None:
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=192,
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(0.0015328)
def test_prompt_tokens_details_without_cached_tokens_details_unchanged(
_local_model_cost_map: None,
) -> None:
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116, audio_tokens=167, cached_tokens=192
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(0.0029888)
def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None:
model_info: ModelInfo = {
"input_cost_per_token": 4e-6,
"input_cost_per_audio_token": 32e-6,
"cache_read_input_token_cost": 5e-7,
}
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=192,
cached_tokens_details={"text_tokens": 64, "audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="some-realtime-model",
usage=usage,
custom_llm_provider="openai",
model_info=model_info,
)
expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7
assert prompt_cost == pytest.approx(expected)
def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None) -> None:
"""Nested cached_tokens_details exceeding cached_tokens must not over-subtract the audio bucket."""
usage = Usage(
prompt_tokens=283,
completion_tokens=0,
total_tokens=283,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=116,
audio_tokens=167,
cached_tokens=100,
cached_tokens_details={"audio_tokens": 128},
),
)
prompt_cost, _ = generic_cost_per_token(
model="gpt-realtime-2", usage=usage, custom_llm_provider="openai"
)
assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7)
def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_lookup(_local_model_cost_map: None) -> None:
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=400,
audio_tokens=600,
cached_tokens=500,
cached_tokens_details={"text_tokens": 100, "audio_tokens": 400},
),
)
prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai")
assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7)
def test_cache_read_breakdown_splits_cached_audio_at_the_audio_cache_rate(_local_model_cost_map: None) -> None:
usage = Usage(
prompt_tokens=4863,
completion_tokens=1087,
total_tokens=5950,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=1693,
audio_tokens=3170,
cached_tokens=2816,
cached_tokens_details={"text_tokens": 896, "audio_tokens": 1920},
),
)
breakdown = get_token_type_cost_breakdown(model="gpt-realtime-2.1-mini", custom_llm_provider="openai", usage=usage)
prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai")
assert breakdown.cache_read_cost == pytest.approx(896 * 6e-8 + 1920 * 3e-7)
assert breakdown.rates is not None
assert breakdown.rates.cache_read_input_audio_token_cost == pytest.approx(3e-7)
assert prompt_cost == pytest.approx((1693 - 896) * 6e-7 + (3170 - 1920) * 1e-5 + breakdown.cache_read_cost)
@pytest.mark.parametrize(
("model", "custom_llm_provider", "expected_prompt_cost"),
(
pytest.param("azure/gpt-realtime-2025-08-28", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime"),
pytest.param("azure/gpt-realtime-1.5-2026-02-23", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime-1.5"),
pytest.param("azure/gpt-realtime-mini", "azure", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="azure-gpt-realtime-mini"),
pytest.param("gpt-realtime-mini", "openai", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="openai-gpt-realtime-mini"),
),
)
def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates(
_local_model_cost_map: None, model: str, custom_llm_provider: str, expected_prompt_cost: float
) -> None:
usage = Usage(
prompt_tokens=1000,
completion_tokens=0,
total_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=400,
audio_tokens=600,
cached_tokens=500,
cached_tokens_details={"text_tokens": 100, "audio_tokens": 400},
),
)
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider)
assert prompt_cost == pytest.approx(expected_prompt_cost)

90
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -5221,10 +5221,11 @@ def test_handle_anthropic_messages_parsed_response_logging_preserves_fast_mode_s
assert getattr(result.usage, "speed", None) == "fast"
def test_logging_init_sets_trace_id():
def test_logging_init_sets_trace_id(monkeypatch):
"""Logging.__init__() must call set_trace_id with self.litellm_trace_id."""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("")
log_obj = Logging(
@ -5240,7 +5241,7 @@ def test_logging_init_sets_trace_id():
assert trace_id_var.get() == log_obj.litellm_trace_id
def test_logging_init_skips_stamping_when_correlation_logging_unsupported():
def test_logging_init_skips_stamping_when_correlation_logging_unsupported(monkeypatch):
"""supports_correlation_logging=False (what wrapper(), the sync entry
point, always passes) must leave trace_id_var/session_id_var completely
untouched, even though self.litellm_trace_id/litellm_session_id (the
@ -5248,6 +5249,7 @@ def test_logging_init_skips_stamping_when_correlation_logging_unsupported():
usual - only the ambient contextvar stamping is gated."""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("")
session_id_var.set("")
@ -5271,10 +5273,48 @@ def test_logging_init_skips_stamping_when_correlation_logging_unsupported():
assert log_obj.litellm_session_id == "should-not-be-stamped"
def test_logging_init_sets_session_id_when_provided():
def test_logging_init_skips_stamping_when_request_correlation_in_logs_disabled(monkeypatch):
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", False)
trace_id_var.set("outer")
session_id_var.set("outer-sid")
try:
with (
patch( # test-quality-ok: regression test verifies disabled stamping skips both setters
"litellm.litellm_core_utils.litellm_logging.set_trace_id"
) as mock_set_trace_id,
patch( # test-quality-ok: regression test verifies disabled stamping skips both setters
"litellm.litellm_core_utils.litellm_logging.set_session_id"
) as mock_set_session_id,
):
log_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="completion",
start_time=None,
litellm_call_id="call-disabled",
function_id="fn-disabled",
kwargs={"litellm_session_id": "disabled-session"},
supports_correlation_logging=True,
)
assert trace_id_var.get() == "outer"
assert session_id_var.get() == "outer-sid"
assert log_obj._own_trace_id == "outer"
mock_set_trace_id.assert_not_called()
mock_set_session_id.assert_not_called()
finally:
trace_id_var.set("")
session_id_var.set("")
def test_logging_init_sets_session_id_when_provided(monkeypatch):
"""Logging.__init__() must call set_session_id when litellm_session_id is in kwargs."""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
session_id_var.set("")
Logging(
@ -5290,11 +5330,12 @@ def test_logging_init_sets_session_id_when_provided():
assert session_id_var.get() == "my-session-99"
def test_logging_init_resets_session_id_to_empty_when_absent():
def test_logging_init_resets_session_id_to_empty_when_absent(monkeypatch):
"""When no session_id is in kwargs, Logging.__init__() must reset session_id_var to ""
so a prior request's session_id does not leak into subsequent log records."""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
session_id_var.set("preexisting-sid")
Logging(
@ -5310,7 +5351,7 @@ def test_logging_init_resets_session_id_to_empty_when_absent():
assert session_id_var.get() == ""
def test_restore_correlation_context_resets_to_pre_call_value():
def test_restore_correlation_context_resets_to_pre_call_value(monkeypatch):
"""_restore_correlation_context() must put trace_id_var/session_id_var back to
whatever they were immediately before this Logging instance was constructed.
This is the mechanism that prevents a nested call (e.g. a guardrail's own
@ -5318,6 +5359,7 @@ def test_restore_correlation_context_resets_to_pre_call_value():
session_id into the outer call's subsequent log lines."""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace")
session_id_var.set("outer-session")
try:
@ -5343,7 +5385,7 @@ def test_restore_correlation_context_resets_to_pre_call_value():
session_id_var.set("")
def test_restore_correlation_context_safe_to_call_repeatedly():
def test_restore_correlation_context_safe_to_call_repeatedly(monkeypatch):
"""Calling _restore_correlation_context() more than once must not raise.
It's deliberately NOT guarded against repeat calls: wrapper()'s finally
@ -5353,6 +5395,7 @@ def test_restore_correlation_context_safe_to_call_repeatedly():
the contextvars, so repeat calls are expected, not just tolerated."""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
log_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "hi"}],
@ -5367,8 +5410,40 @@ def test_restore_correlation_context_safe_to_call_repeatedly():
log_obj._restore_correlation_context() # must not raise
def test_restore_correlation_context_does_not_resanitize(monkeypatch):
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm._logging import _sanitize_correlation_id
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace")
session_id_var.set("outer-session")
try:
log_obj = Logging(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="completion",
start_time=None,
litellm_call_id="call-no-resanitize",
function_id="fn-no-resanitize",
kwargs={"litellm_session_id": "inner-session"},
)
with patch( # test-quality-ok: regression test verifies restore avoids sanitization
"litellm._logging._sanitize_correlation_id", wraps=_sanitize_correlation_id
) as mock_sanitize:
log_obj._restore_correlation_context()
mock_sanitize.assert_not_called()
assert trace_id_var.get() == "outer-trace"
assert session_id_var.get() == "outer-session"
finally:
trace_id_var.set("")
session_id_var.set("")
@pytest.mark.asyncio
async def test_restore_correlation_context_works_across_asyncio_task_boundary():
async def test_restore_correlation_context_works_across_asyncio_task_boundary(monkeypatch):
"""_restore_correlation_context() must succeed even when it's called from a
different asyncio Task than the one Logging.__init__() ran in - exactly what
happens on litellm's real async success path, where async_success_handler is
@ -5385,6 +5460,7 @@ async def test_restore_correlation_context_works_across_asyncio_task_boundary():
"""
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-cross-task")
session_id_var.set("outer-session-cross-task")
try:

24
tests/test_litellm/litellm_core_utils/test_streaming_handler.py
View file

@ -4100,7 +4100,7 @@ async def test_async_streaming_completion_does_not_reset_context_before_iteratio
session_id_var.set("")
def test_stream_wrapper_del_restores_correlation_context():
def test_stream_wrapper_del_restores_correlation_context(monkeypatch):
"""CustomStreamWrapper.__del__ is the best-effort fallback for an abandoned
stream (caller never exhausts it, so the normal terminal-handler restore
never fires). Testing this via real garbage collection is unreliable in
@ -4112,6 +4112,7 @@ def test_stream_wrapper_del_restores_correlation_context():
doesn't run actual finalization, and this exercises exactly the logic that
real garbage collection would eventually trigger.
"""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-abandoned")
session_id_var.set("outer-session-abandoned")
try:
@ -4159,12 +4160,13 @@ def test_stream_wrapper_del_never_raises_with_broken_logging_obj():
wrapper.__del__() # must not raise
def test_stream_wrapper_del_does_not_clobber_a_newer_active_call():
def test_stream_wrapper_del_does_not_clobber_a_newer_active_call(monkeypatch):
"""A delayed finalizer must never stomp a different, still-active call's
context. If an abandoned stream's __del__ fires late - after a new call
has already started in the same Task/thread and claimed the contextvars -
unconditionally restoring the abandoned stream's own pre-call snapshot
would corrupt the active call's subsequent log lines with stale ids."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-before-abandoned-call")
session_id_var.set("outer-session-before-abandoned-call")
try:
@ -4210,13 +4212,14 @@ def test_stream_wrapper_del_does_not_clobber_a_newer_active_call():
session_id_var.set("")
def test_stream_wrapper_del_restores_when_own_session_id_needed_sanitizing():
def test_stream_wrapper_del_restores_when_own_session_id_needed_sanitizing(monkeypatch):
"""The __del__ guard must compare against the *sanitized* id actually
stored in the contextvar, not the raw litellm_session_id/litellm_trace_id
- set_session_id()/set_trace_id() strip control characters before
storing, so a caller-supplied id containing e.g. a newline would never
equal the raw attribute, and the guard would wrongly conclude some other
call has claimed the context and skip cleanup forever."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-needs-sanitizing")
session_id_var.set("outer-session-needs-sanitizing")
try:
@ -4250,7 +4253,7 @@ def test_stream_wrapper_del_restores_when_own_session_id_needed_sanitizing():
session_id_var.set("")
def test_stream_wrapper_next_keeps_context_active_through_synthesized_finish_reason_chunk():
def test_stream_wrapper_next_keeps_context_active_through_synthesized_finish_reason_chunk(monkeypatch):
"""When the underlying stream ends without ever emitting an explicit
finish_reason chunk, __next__ synthesizes one via finish_reason_handler()
and returns it. That chunk is still this call's own data - the caller's
@ -4261,6 +4264,7 @@ def test_stream_wrapper_next_keeps_context_active_through_synthesized_finish_rea
correct, deterministic restore on the very next __next__() call, since
completion_stream is already exhausted and immediately re-raises
StopIteration."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-finish-reason")
session_id_var.set("outer-session-finish-reason")
try:
@ -4300,12 +4304,13 @@ def test_stream_wrapper_next_keeps_context_active_through_synthesized_finish_rea
session_id_var.set("")
def test_stream_wrapper_del_cleans_up_after_synthesized_finish_reason_chunk():
def test_stream_wrapper_del_cleans_up_after_synthesized_finish_reason_chunk(monkeypatch):
"""A caller that breaks immediately after seeing finish_reason (the
early-break pattern) never triggers the next()-driven restore above - it
relies on the best-effort __del__ guard instead, same as any other
abandoned stream. The guard must still recognize this call's own
(unrestored) ids as unclaimed and clean them up."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-finish-reason-del")
session_id_var.set("outer-session-finish-reason-del")
try:
@ -4338,10 +4343,11 @@ def test_stream_wrapper_del_cleans_up_after_synthesized_finish_reason_chunk():
@pytest.mark.asyncio
async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_finish_reason_chunk():
async def test_stream_wrapper_anext_keeps_context_active_through_synthesized_finish_reason_chunk(monkeypatch):
"""Async sibling of test_stream_wrapper_next_keeps_context_active_through_synthesized_finish_reason_chunk -
_finalize_completed_stream()'s else branch must not restore before
returning the synthesized chunk either."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-anext-finish-reason")
session_id_var.set("outer-session-anext-finish-reason")
try:
@ -4394,6 +4400,7 @@ async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_corre
path as every other failure so the consumer's outer correlation context gets
restored - calling the check before entering __anext__()'s try block would
let the Timeout bypass that restoration entirely."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
monkeypatch.setattr(litellm.constants, "LITELLM_MAX_STREAMING_DURATION_SECONDS", 1)
trace_id_var.set("outer-trace-max-duration")
session_id_var.set("outer-session-max-duration")
@ -4434,12 +4441,13 @@ async def test_stream_wrapper_anext_max_duration_timeout_restores_consumer_corre
@pytest.mark.asyncio
async def test_stream_wrapper_aclose_restores_consumer_correlation_context():
async def test_stream_wrapper_aclose_restores_consumer_correlation_context(monkeypatch):
"""Explicit early termination (aclose(), e.g. on client disconnect or a
router fallback aborting an in-progress stream) must restore the caller's
correlation context too - not just __del__'s best-effort GC-timed fallback,
since aclose() is normally called deterministically by the consumer/
framework, unlike __del__."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-aclose")
session_id_var.set("outer-session-aclose")
try:
@ -4481,6 +4489,7 @@ async def test_stream_wrapper_aclose_keeps_context_active_through_close_failure_
branch logs a debug diagnostic. That log line must still carry the
closing stream's own trace_id/session_id - the outer context must not be
restored until after the close attempt (and its diagnostic) completes."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-close-fail")
session_id_var.set("outer-session-close-fail")
try:
@ -4541,6 +4550,7 @@ def test_handle_stream_fallback_error_restores_context_only_after_exception_mapp
mapping. The consumer's outer context must not be restored until that
mapping call returns, or the diagnostic log line would carry the outer
(or empty) trace_id/session_id instead of the failing stream's own."""
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
trace_id_var.set("outer-trace-fallback")
session_id_var.set("outer-session-fallback")
try:

225
tests/test_litellm/proxy/auth/test_user_api_key_auth.py
View file

@ -13,7 +13,7 @@ from unittest.mock import ANY, AsyncMock, MagicMock, patch
import pytest
from fastapi import status
from fastapi import HTTPException, status
import litellm
import litellm.proxy.proxy_server
@ -7698,3 +7698,226 @@ async def test_claude_view_never_reinterprets_explicit_names(monkeypatch, layer)
assert data["model"] == ("foo" if layer == "unclaimed" else encoded)
await _normalize_claude_model(data, token, request, "/v1/messages")
assert data["model"] == ("foo" if layer == "unclaimed" else encoded)
ISSUER_ONE = "https://issuer-one.example.com"
ISSUER_TWO = "https://issuer-two.example.com"
def _per_issuer_virtual_key_jwt_handler(
global_claim_field: str | None, global_behavior: str = "fallback_team_mapping"
) -> MagicMock:
jwt_handler = MagicMock()
jwt_handler.is_jwt.return_value = True
jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(
virtual_key_claim_field=global_claim_field,
unregistered_jwt_client_behavior=global_behavior,
issuers=[
{
"issuer": ISSUER_ONE,
"jwks_url": f"{ISSUER_ONE}/keys",
"audience": "audience-one",
"team_id_jwt_field": "sub",
},
{
"issuer": ISSUER_TWO,
"jwks_url": f"{ISSUER_TWO}/keys",
"audience": "audience-two",
"virtual_key_claim_field": "sub",
"unregistered_jwt_client_behavior": "reject",
},
],
)
return jwt_handler
def _fake_prisma_with_jwt_key_mapping(hashed_token: str | None) -> tuple[SimpleNamespace, AsyncMock]:
find_first = AsyncMock(return_value=None if hashed_token is None else SimpleNamespace(token=hashed_token))
prisma_client = SimpleNamespace(db=SimpleNamespace(litellm_jwtkeymapping=SimpleNamespace(find_first=find_first)))
return prisma_client, find_first
def _mapping_where(claim_name: str, claim_value: str) -> dict[str, str | bool]:
return {"jwt_claim_name": claim_name, "jwt_claim_value": claim_value, "is_active": True}
@pytest.mark.asyncio
async def test_per_issuer_virtual_key_claim_field_selects_the_issuer_mapping_for_the_db_lookup():
from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key
jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field=None)
prisma_client, find_first = _fake_prisma_with_jwt_key_mapping("hashed-mapped-key")
user_api_key_cache = DualCache()
await user_api_key_cache.async_set_cache(
key="hashed-mapped-key",
value=UserAPIKeyAuth(token="hashed-mapped-key", api_key="hashed-mapped-key", team_id="svc-team"),
)
resolved = await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "svc-account-7"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
find_first.assert_awaited_once_with(where=_mapping_where("sub", "svc-account-7"))
assert isinstance(resolved, UserAPIKeyAuth)
assert resolved.token == "hashed-mapped-key"
assert resolved.team_id == "svc-team"
assert await user_api_key_cache.async_get_cache("jwt_key_mapping:sub:svc-account-7") == "hashed-mapped-key"
@pytest.mark.asyncio
async def test_per_issuer_reject_behavior_does_not_leak_into_the_team_issuer():
from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key
jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field=None)
prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None)
team_issuer_result = await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "team-alpha"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=DualCache(),
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
assert team_issuer_result is None
find_first.assert_not_awaited()
with pytest.raises(HTTPException) as exc:
await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "unknown-svc"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=DualCache(),
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
assert exc.value.status_code == 403
assert "No registered mapping for sub='unknown-svc'" in str(exc.value.detail)
find_first.assert_awaited_once_with(where=_mapping_where("sub", "unknown-svc"))
@pytest.mark.asyncio
async def test_proxy_admin_sentinel_cached_by_another_issuer_does_not_bypass_reject():
from litellm.proxy.auth.user_api_key_auth import _JWT_PROXY_ADMIN_SENTINEL, _resolve_jwt_to_virtual_key
jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="sub", global_behavior="auto_register")
prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None)
user_api_key_cache = DualCache()
await user_api_key_cache.async_set_cache(key="jwt_key_mapping:sub:admin-7", value=_JWT_PROXY_ADMIN_SENTINEL)
auto_register_issuer_result = await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "admin-7"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
assert auto_register_issuer_result is None
find_first.assert_not_awaited()
with pytest.raises(HTTPException) as exc:
await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "admin-7"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
assert exc.value.status_code == 403
assert "No registered mapping for sub='admin-7'" in str(exc.value.detail)
find_first.assert_awaited_once_with(where=_mapping_where("sub", "admin-7"))
@pytest.mark.asyncio
async def test_issuer_without_virtual_key_claim_field_falls_back_to_the_global_field():
from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key
jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="client_id")
prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None)
with_claim = await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "team-alpha", "client_id": "app-9"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=DualCache(),
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
without_claim = await _resolve_jwt_to_virtual_key(
jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "team-alpha"},
jwt_handler=jwt_handler,
prisma_client=prisma_client,
user_api_key_cache=DualCache(),
parent_otel_span=None,
proxy_logging_obj=MagicMock(),
)
assert with_claim is None
assert without_claim is None
find_first.assert_awaited_once_with(where=_mapping_where("client_id", "app-9"))
@pytest.mark.asyncio
async def test_auth_flow_enters_virtual_key_mapping_when_only_an_issuer_configures_the_claim_field():
jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJzdmMtYWNjb3VudC03In0.signature"
jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field=None)
jwt_handler.auth_jwt = AsyncMock(
return_value={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "svc-account-7"}
)
mapped_key = UserAPIKeyAuth(token="hashed-mapped-key", api_key="hashed-mapped-key", team_id="svc-team")
mock_request = MagicMock()
mock_request.url.path = "/v1/chat/completions"
mock_request.method = "POST"
mock_request.headers = {"authorization": f"Bearer {jwt_token}"}
mock_request.query_params = {}
mock_request.state = SimpleNamespace()
with (
patch( # test-quality-ok: the builder reads proxy settings from module globals, no injection seam
"litellm.proxy.proxy_server.general_settings", {"enable_jwt_auth": True}
),
patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: module-global proxy state
patch("litellm.proxy.proxy_server.master_key", "sk-master"), # test-quality-ok: module-global proxy state
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: module-global proxy state
patch( # test-quality-ok: module-global proxy state
"litellm.proxy.proxy_server.user_api_key_cache", DualCache()
),
patch( # test-quality-ok: module-global proxy state
"litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()
),
patch("litellm.proxy.proxy_server.jwt_handler", jwt_handler), # test-quality-ok: module-global proxy state
patch( # test-quality-ok: the regression is whether the builder reaches this seam at all
"litellm.proxy.auth.user_api_key_auth._resolve_jwt_to_virtual_key",
new_callable=AsyncMock,
return_value=mapped_key,
) as resolve_mock,
patch( # test-quality-ok: a mapped key must short-circuit standard JWT auth; reaching it is the failure
"litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder",
new_callable=AsyncMock,
side_effect=AssertionError("standard JWT auth must not run for a mapped virtual key"),
),
):
result = await _user_api_key_auth_builder(
request=mock_request,
api_key=jwt_token,
azure_api_key_header="",
anthropic_api_key_header=None,
google_ai_studio_api_key_header=None,
azure_apim_header=None,
request_data={"model": "gpt-4o-mini"},
)
resolve_mock.assert_awaited_once()
assert resolve_mock.await_args.kwargs["jwt_claims"][JWTHandler.LITELLM_JWT_ISSUER_CLAIM] == ISSUER_TWO
assert result.api_key == "hashed-mapped-key"
assert result.team_id == "svc-team"

58
tests/test_litellm/proxy/test__types.py
View file

@ -277,3 +277,61 @@ def test_team_membership_budget_table_present_still_works():
}
result = LiteLLM_TeamMembership.model_validate(data)
assert result.litellm_budget_table is None
def test_a_jwt_issuer_can_override_the_virtual_key_claim_field_while_other_issuers_keep_the_global_one():
from litellm.proxy._types import LiteLLM_JWTAuth, UnregisteredJWTClientBehavior
jwt_auth = LiteLLM_JWTAuth(
virtual_key_claim_field="client_id",
issuers=[
{
"issuer": "https://team-idp.example.com",
"jwks_url": "https://team-idp.example.com/keys",
"audience": "litellm",
"team_id_jwt_field": "sub",
},
{
"issuer": "https://service-idp.example.com",
"jwks_url": "https://service-idp.example.com/keys",
"audience": "litellm",
"virtual_key_claim_field": "sub",
"unregistered_jwt_client_behavior": "reject",
},
],
)
assert jwt_auth.get_virtual_key_claim_field("https://service-idp.example.com") == "sub"
assert jwt_auth.get_unregistered_jwt_client_behavior("https://service-idp.example.com") is (
UnregisteredJWTClientBehavior.REJECT
)
assert jwt_auth.get_virtual_key_claim_field("https://team-idp.example.com") == "client_id"
assert jwt_auth.get_unregistered_jwt_client_behavior("https://team-idp.example.com") is (
UnregisteredJWTClientBehavior.FALLBACK_TEAM_MAPPING
)
assert jwt_auth.get_virtual_key_claim_field(None) == "client_id"
assert jwt_auth.get_virtual_key_claim_field("https://unknown-idp.example.com") == "client_id"
@pytest.mark.parametrize(
("global_field", "issuer_field", "is_configured"),
((None, None, False), ("sub", None, True), (None, "sub", True)),
)
def test_virtual_key_mapping_counts_as_configured_when_any_issuer_sets_the_claim_field(
global_field, issuer_field, is_configured
):
from litellm.proxy._types import LiteLLM_JWTAuth
jwt_auth = LiteLLM_JWTAuth(
virtual_key_claim_field=global_field,
issuers=[
{
"issuer": "https://idp.example.com",
"jwks_url": "https://idp.example.com/keys",
"audience": "litellm",
"virtual_key_claim_field": issuer_field,
}
],
)
assert jwt_auth.is_virtual_key_mapping_configured() is is_configured

2
tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -2789,6 +2789,7 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 5
assert response_usage.input_tokens_details.text_tokens == 8
assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump()
def test_transform_usage_with_cached_tokens_gemini(self):
"""Test that cached_tokens from Gemini are properly transformed to input_tokens_details"""
@ -2851,6 +2852,7 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 100
assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800
assert response_usage.input_tokens_details.model_dump()["cache_write_tokens"] == 800
def test_transform_usage_with_reasoning_tokens_gemini(self):
"""Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details"""

41
tests/test_litellm/responses/test_responses_utils.py
View file

@ -577,6 +577,47 @@ class TestResponseAPILoggingUtils:
assert result.completion_tokens_details is not None
assert result.completion_tokens_details.reasoning_tokens == 4
def test_transform_realtime_usage_dict_keeps_cached_tokens_details(self):
usage = {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens == 192
assert result.prompt_tokens_details.cached_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
def test_transform_response_api_usage_object_keeps_cached_tokens_details(self):
usage = ResponseAPIUsage(
input_tokens=283,
output_tokens=0,
total_tokens=283,
input_tokens_details={
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
)
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.prompt_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details is not None
assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64
class TestResponsesAPIProviderSpecificParams:
"""

104
tests/test_litellm/test_cost_calculator.py
View file

@ -19,6 +19,7 @@ from litellm.cost_calculator import (
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.openai import OpenAIRealtimeStreamList
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
@ -4848,6 +4849,109 @@ def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() ->
assert combined.completion_tokens_details.audio_tokens == 0
def test_realtime_combine_sums_nested_cached_tokens_details():
results: OpenAIRealtimeStreamList = [
{
"type": "response.done",
"response": {
"usage": {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
},
},
{
"type": "response.done",
"response": {
"usage": {
"input_tokens": 150,
"output_tokens": 0,
"total_tokens": 150,
"input_token_details": {
"text_tokens": 50,
"audio_tokens": 100,
"cached_tokens": 100,
"cached_tokens_details": {"audio_tokens": 100},
},
}
},
},
]
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
results=results,
)
assert combined.prompt_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens == 292
assert combined.prompt_tokens_details.cached_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens_details.audio_tokens == 228
assert combined.prompt_tokens_details.cached_tokens_details.text_tokens == 64
assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None
@pytest.mark.parametrize("details_first", [True, False])
def test_realtime_combine_keeps_cached_split_when_only_one_usage_has_details(details_first: bool):
with_details: Final = {
"type": "response.done",
"response": {
"usage": {
"input_tokens": 283,
"output_tokens": 0,
"total_tokens": 283,
"input_token_details": {
"text_tokens": 116,
"audio_tokens": 167,
"cached_tokens": 192,
"cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128},
},
}
},
}
without_details: Final = {
"type": "response.done",
"response": {
"usage": {
"input_tokens": 150,
"output_tokens": 0,
"total_tokens": 150,
"input_token_details": {"text_tokens": 50, "audio_tokens": 100, "cached_tokens": 100},
}
},
}
results: OpenAIRealtimeStreamList = (
[with_details, without_details] if details_first else [without_details, with_details]
)
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(
results=results,
)
assert combined.prompt_tokens_details is not None
assert combined.prompt_tokens_details.cached_tokens == 292
assert combined.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=64, audio_tokens=128)
def test_usage_without_cached_tokens_details_omits_key():
usage = Usage(
prompt_tokens=10,
completion_tokens=5,
total_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10),
)
dumped = usage.prompt_tokens_details.model_dump()
assert "cached_tokens_details" not in dumped
assert "cached_tokens_details" not in usage.prompt_tokens_details.model_dump_json()
UNMAPPED_OCR_MODEL: Final = "azure_ai/some-unmapped-ocr-model-for-testing"
MAPPED_OCR_MODEL: Final = "mistral/mistral-ocr-4-0"

8
tests/test_litellm/test_model_prices_schema.py
View file

@ -221,6 +221,14 @@ def test_chat_latest_declares_the_one_effort_openai_accepts(prices: dict):
assert resolve_supported_reasoning_efforts(prices["chat-latest"], deployment_is_mapped=True) == ("medium",)
@pytest.mark.parametrize("key", ["azure/gpt-chat-latest", "azure/chat-latest", "azure/us/gpt-chat-latest"])
def test_azure_gpt_chat_latest_declares_the_one_effort_azure_accepts(prices: dict, key: str):
"""Azure answers every reasoning_effort on a gpt-chat-latest deployment except medium with
"Unsupported value ... Supported values are: 'medium'", the same fixed level OpenAI's chat-latest
carries, so the Foundry product name and the OpenAI API name both declare that one level."""
assert resolve_supported_reasoning_efforts(prices[key], deployment_is_mapped=True) == ("medium",)
BEDROCK_OPENAI_GPT_MARKERS: Final = ("openai.gpt-5.4", "openai.gpt-5.5", "openai.gpt-5.6", "openai.gpt-6-astra")
BEDROCK_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse", "bedrock_mantle"))
BEDROCK_ROW_PREFIXES: Final = ("bedrock_mantle/", "us.", "global.")

8
tests/test_litellm/test_utils.py
View file

@ -6455,3 +6455,11 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th
assert snapshot["litellm_call_id"]
assert snapshot["response_cost"] is not None
assert snapshot["api_base"]
def test_get_model_info_carries_cache_read_input_audio_token_cost(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
info = litellm.get_model_info("gpt-realtime-2.1-mini", custom_llm_provider="openai")
assert info["cache_read_input_audio_token_cost"] == 3e-07
assert info["cache_read_input_token_cost"] == 6e-08

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -16781,7 +16781,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -16887,7 +16886,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)