From 492251c7bc7bd76baf25512e4c43421efbcff799 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 10 Sep 2026 07:32:54 +0000 Subject: [PATCH 001/114] fix(otel): cap per-index OpenInference message attributes span-wide OpenInferenceMapper spelled every captured prompt and response message out as two indexed attributes with no bound. A few dozen turns overran the OTel SDK's 128-attribute span limit, which evicts oldest first, so the gen_ai.* model, provider, usage, cost and finish reason written before it were what got dropped. Both directions now share one MAX_MESSAGE_ATTRS_PER_SPAN ceiling, the response keeps at least half of it, and input.value / output.value still carry the complete conversation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../otel/mappers/openinference.py | 26 ++- litellm/integrations/otel/mappers/utils.py | 12 ++ .../integrations/otel/test_otel_v2_emitter.py | 148 ++++++++++++++++++ 3 files changed, 181 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/otel/mappers/openinference.py b/litellm/integrations/otel/mappers/openinference.py index 0ba45170b8e..1e2dbf6974d 100644 --- a/litellm/integrations/otel/mappers/openinference.py +++ b/litellm/integrations/otel/mappers/openinference.py @@ -12,6 +12,7 @@ from typing import Final from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData from litellm.integrations.otel.mappers.utils import ( + MAX_MESSAGE_ATTRS_PER_SPAN, MAX_TOOL_DEFINITION_ATTRS_PER_SPAN, collect, drop_none, @@ -26,6 +27,8 @@ from litellm.integrations.otel.model.payloads import ( ToolDefinition, ) +_MAX_INDEXED_MESSAGES: Final = MAX_MESSAGE_ATTRS_PER_SPAN // 2 + class OpenInferenceMapper: """Emits OpenInference attributes for LLM_CALL spans. @@ -84,22 +87,35 @@ class OpenInferenceMapper: return {} def _llm_call(self, data: LLMCallSpanData) -> AttributeMap: + outputs: Final = output_messages(data) + indexed_in, indexed_out = self._indexed_split(len(data.messages_in), len(outputs)) return { **collect(self._LLM_CALL_ATTRS, data), **collect(self._BLOB_ATTRS, data), - **self._messages("llm.input_messages", "input.value", data.messages_in), - **self._messages("llm.output_messages", "output.value", output_messages(data)), + **self._messages("llm.input_messages", "input.value", data.messages_in, indexed_in), + **self._messages("llm.output_messages", "output.value", outputs, indexed_out), **self._tools(data), } @staticmethod - def _messages(prefix: str, value_key: str, messages: Sequence[object]) -> AttributeMap: - """Per-message ``{prefix}.{idx}.message.*`` keys + the ``value_key`` blob.""" + def _indexed_split(inputs: int, outputs: int) -> tuple[int, int]: + """How many prompt and response messages get per-index attributes. + + Both directions share one span-wide allowance. The response is reserved at + least half of it, so a long prompt can never push the completion off the + span, and the prompt takes whatever the response leaves unused. + """ + indexed_out: Final = min(outputs, max(_MAX_INDEXED_MESSAGES // 2, _MAX_INDEXED_MESSAGES - inputs)) + return _MAX_INDEXED_MESSAGES - indexed_out, indexed_out + + @staticmethod + def _messages(prefix: str, value_key: str, messages: Sequence[object], indexed: int) -> AttributeMap: + """``{prefix}.{idx}.message.*`` keys for the leading ``indexed`` messages + the ``value_key`` blob of all.""" parsed: Final = [(m.get("role") if isinstance(m, dict) else None, message_content(m)) for m in messages] attrs: Final = drop_none( { key: value - for idx, (role, content) in enumerate(parsed) + for idx, (role, content) in enumerate(parsed[:indexed]) for key, value in ( ( f"{prefix}.{idx}.message.role", diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index d45dca782b2..8d21b774319 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -32,6 +32,18 @@ core telemetry no matter how many vocabularies are configured. """ +MAX_MESSAGE_ATTRS_PER_SPAN: Final = DEFAULT_SPAN_ATTRIBUTE_LIMIT // 4 +"""Span-wide ceiling on attributes spent spelling out chat messages per index. + +A conversation is the other unbounded family: two attributes per message, for +the prompt and the response alike, on the same span. Past a few dozen turns the +family alone exceeds the span attribute limit and evicts the core telemetry +written before it. The ceiling covers both directions together, since a budget +handed to each direction separately doubles. The complete conversation still +rides the JSON blob attributes; only the per-index convenience keys are capped. +""" + + def tool_attr_budget(vocabularies: int) -> int: """Split the span-wide tool-definition ceiling across active vocabularies.""" return MAX_TOOL_DEFINITION_ATTRS_PER_SPAN // max(vocabularies, 1) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py index b1b1b62c820..a417bd62124 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py @@ -1,6 +1,8 @@ """Golden tests for the OTel v2 engine: span shape, kinds, semconv attributes, legacy dual-emit, hierarchy, error status, and idempotency. Needs the OTel SDK.""" +import json + import pytest pytest.importorskip("opentelemetry") @@ -18,6 +20,7 @@ from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.emitter import SpanEmitter # noqa: E402 from litellm.integrations.otel.emitter import stamp_error # noqa: E402 from litellm.integrations.otel.mappers.utils import ( # noqa: E402 + MAX_MESSAGE_ATTRS_PER_SPAN, MAX_TOOL_DEFINITION_ATTRS_PER_SPAN, ) from litellm.integrations.otel.model.payloads import ( # noqa: E402 @@ -440,3 +443,148 @@ def test_vendor_tool_definitions_are_truncated_not_dropped(): assert a["llm.tools.0.tool.name"] == "tool_0" assert a["llm.tools.0.tool.json_schema"] assert "llm.tools.126.tool.name" not in a + + +def _conversation_payload(turns, choices=1, **overrides): + """A ``turns``-message chat with ``choices`` response choices, content-bearing.""" + return _payload( + messages=[{"role": ("user", "assistant")[i % 2], "content": f"turn {i}"} for i in range(turns)], + response={ + "id": "resp_1", + "model": "gpt-4o-2024", + "choices": [ + {"finish_reason": "stop", "message": {"role": "assistant", "content": f"reply {i}"}} + for i in range(choices) + ], + }, + **overrides, + ) + + +def _conversation_span(mapper_names, payload): + """The exported LLM-call span for ``payload`` with content capture on.""" + cfg = OpenTelemetryV2Config( + exporter="in_memory", + mapper_names=list(mapper_names), + capture_message_content="span_only", + ) + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "litellm-test"), cfg) + engine.emit( + SpanRole.LLM_CALL, + LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True), + ) + (span,) = exporter.get_finished_spans() + return span + + +def _indexed_message_count(attributes, prefix): + return len({key.split(".")[2] for key in attributes if key.startswith(f"{prefix}.")}) + + +@pytest.mark.parametrize("turns", [60, 200]) +def test_long_conversation_does_not_evict_core_attributes(turns): + """Per-message OpenInference attributes must never crowd core telemetry off the span. + + With content capture on, the OpenInference vocabulary spells every prompt and + response message out as two per-index attributes. A few dozen turns overruns + the OTel SDK's 128-attribute span limit, which evicts oldest-first, so the + ``gen_ai.*`` set written before it is what disappears. + """ + span = _conversation_span(["genai", "openinference"], _conversation_payload(turns)) + a = span.attributes + + assert span.dropped_attributes == 0 + assert a[GenAI.REQUEST_MODEL] == "gpt-4o" + assert a[GenAI.PROVIDER_NAME] == "openai" + assert a[GenAI.USAGE_INPUT_TOKENS] == 10 + assert a[GenAI.USAGE_OUTPUT_TOKENS] == 5 + assert a[GenAI.RESPONSE_FINISH_REASONS] == ("stop",) + assert a[f"{LiteLLM.COST_PREFIX}total"] == 0.002 + + assert a["llm.input_messages.0.message.content"] == "turn 0" + assert a["llm.output_messages.0.message.content"] == "reply 0" + assert f"llm.input_messages.{turns - 1}.message.role" not in a + assert len(json.loads(a["input.value"])) == turns + assert len(json.loads(a["output.value"])) == 1 + assert len(json.loads(a[GenAI.INPUT_MESSAGES])) == turns + + +def test_short_conversation_keeps_every_message_indexed(): + """Below the cap nothing is truncated in either direction.""" + a = _conversation_span(["genai", "openinference"], _conversation_payload(4, choices=2)).attributes + for idx in range(4): + assert a[f"llm.input_messages.{idx}.message.content"] == f"turn {idx}" + for idx in range(2): + assert a[f"llm.output_messages.{idx}.message.content"] == f"reply {idx}" + + +def test_message_cap_is_shared_across_input_and_output(): + """One span-wide allowance covers both directions, and the response always keeps a share. + + A long prompt takes what a single reply leaves over, and a many-choice reply + cannot take the whole allowance away from the prompt either. + """ + long_prompt = _conversation_span(["genai", "openinference"], _conversation_payload(60, choices=1)).attributes + many_choices = _conversation_span(["genai", "openinference"], _conversation_payload(60, choices=20)).attributes + + single_reply_indexed = _indexed_message_count(long_prompt, "llm.output_messages") + assert single_reply_indexed == 1 + assert _indexed_message_count(long_prompt, "llm.input_messages") + single_reply_indexed == ( + MAX_MESSAGE_ATTRS_PER_SPAN // 2 + ) + + assert _indexed_message_count(many_choices, "llm.input_messages") > 0 + assert _indexed_message_count(many_choices, "llm.output_messages") > single_reply_indexed + assert _indexed_message_count(many_choices, "llm.input_messages") + _indexed_message_count( + many_choices, "llm.output_messages" + ) == (MAX_MESSAGE_ATTRS_PER_SPAN // 2) + + +def test_fully_populated_arize_span_stays_within_the_attribute_limit(): + """Every capped family maxed at once still leaves the whole core intact. + + The Arize / Phoenix composition (``genai`` + ``openinference`` + ``legacy``) + with every request parameter, every cost component, a hundred-plus tools, a + two-hundred-turn prompt and twenty choices is the worst case the two + span-wide ceilings have to absorb together. + """ + payload = _conversation_payload( + 200, + choices=20, + stream=True, + model_parameters={ + **_tools_payload(127)["model_parameters"], + "top_p": 0.9, + "frequency_penalty": 0.1, + "presence_penalty": 0.1, + "seed": 7, + "stop": ["\n"], + }, + cost_breakdown={ + key: 0.001 + for key in ( + "input_cost", + "output_cost", + "cache_read_cost", + "cache_creation_cost", + "tool_usage_cost", + "original_cost", + "discount_amount", + "discount_percent", + "margin_fixed_amount", + "margin_percent", + "margin_total_amount", + "total_cost", + ) + }, + ) + span = _conversation_span(["genai", "openinference"], payload) + a = span.attributes + + assert span.dropped_attributes == 0 + assert a[GenAI.REQUEST_MODEL] == "gpt-4o" + assert a[f"{LiteLLM.COST_PREFIX}total"] == 0.002 + assert a[LiteLLM.TOOLS_DECLARED] == 127 + assert a["llm.input_messages.0.message.content"] == "turn 0" + assert a["llm.output_messages.0.message.content"] == "reply 0" From fcaf2d7d98164fc8561c411a999ffd42469797e9 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 10 Sep 2026 08:00:51 +0000 Subject: [PATCH 002/114] fix(otel): size the message ceiling so every vocabulary fits beside it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/otel/mappers/utils.py | 9 ++++++--- .../integrations/otel/test_otel_v2_emitter.py | 15 ++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index 8d21b774319..d8918491720 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -32,15 +32,18 @@ core telemetry no matter how many vocabularies are configured. """ -MAX_MESSAGE_ATTRS_PER_SPAN: Final = DEFAULT_SPAN_ATTRIBUTE_LIMIT // 4 +MAX_MESSAGE_ATTRS_PER_SPAN: Final = DEFAULT_SPAN_ATTRIBUTE_LIMIT // 8 """Span-wide ceiling on attributes spent spelling out chat messages per index. A conversation is the other unbounded family: two attributes per message, for the prompt and the response alike, on the same span. Past a few dozen turns the family alone exceeds the span attribute limit and evicts the core telemetry written before it. The ceiling covers both directions together, since a budget -handed to each direction separately doubles. The complete conversation still -rides the JSON blob attributes; only the per-index convenience keys are capped. +handed to each direction separately doubles. An eighth is the largest share +that still fits beside the tool ceiling and the core of every vocabulary at +once, request parameters, cost breakdown and identity included. The complete +conversation still rides the JSON blob attributes; only the per-index +convenience keys are capped. """ diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py index a417bd62124..f571ab7004b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py @@ -461,10 +461,11 @@ def _conversation_payload(turns, choices=1, **overrides): ) -def _conversation_span(mapper_names, payload): +def _conversation_span(mapper_names, payload, legacy_compat=False): """The exported LLM-call span for ``payload`` with content capture on.""" cfg = OpenTelemetryV2Config( exporter="in_memory", + legacy_compat=legacy_compat, mapper_names=list(mapper_names), capture_message_content="span_only", ) @@ -541,13 +542,13 @@ def test_message_cap_is_shared_across_input_and_output(): ) == (MAX_MESSAGE_ATTRS_PER_SPAN // 2) -def test_fully_populated_arize_span_stays_within_the_attribute_limit(): +def test_fully_populated_span_with_every_vocabulary_stays_within_the_attribute_limit(): """Every capped family maxed at once still leaves the whole core intact. - The Arize / Phoenix composition (``genai`` + ``openinference`` + ``legacy``) - with every request parameter, every cost component, a hundred-plus tools, a - two-hundred-turn prompt and twenty choices is the worst case the two - span-wide ceilings have to absorb together. + Every vocabulary in the registry plus ``legacy``, every request parameter, + every cost component, a hundred-plus tools, a two-hundred-turn prompt and + twenty choices is the worst case the two span-wide ceilings have to absorb + together. """ payload = _conversation_payload( 200, @@ -579,7 +580,7 @@ def test_fully_populated_arize_span_stays_within_the_attribute_limit(): ) }, ) - span = _conversation_span(["genai", "openinference"], payload) + span = _conversation_span(["genai", "openinference", "langfuse", "weave", "langtrace"], payload, legacy_compat=True) a = span.attributes assert span.dropped_attributes == 0 From 302a8d43da054f98b7ccb75baf43a8a79bb3ea40 Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 10 Sep 2026 22:02:58 +0000 Subject: [PATCH 003/114] fix(cost): bill cached realtime audio tokens at the audio cache-read rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 78 +++++++++++++------ .../litellm_core_utils/llm_cost_calc/utils.py | 40 ++++++++-- ...odel_prices_and_context_window_backup.json | 3 + .../transformation.py | 45 ++++++----- litellm/responses/utils.py | 3 + litellm/types/llms/openai.py | 14 ++++ litellm/types/utils.py | 6 ++ model_prices_and_context_window.json | 3 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 68 ++++++++++++++++ .../responses/test_responses_utils.py | 41 ++++++++++ tests/test_litellm/test_cost_calculator.py | 61 +++++++++++++++ 11 files changed, 314 insertions(+), 48 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 814eaaf76f7..7cd3ea8f303 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -108,6 +108,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( + CachedTokensDetails, CallTypesLiteral, LiteLLMRealtimeStreamLoggingObject, LlmProviders, @@ -2310,6 +2311,60 @@ def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str] return [attr for attr in field_names if attr != "cache_creation_tokens"] +def _combine_cached_tokens_details( + current: CachedTokensDetails | None, new: CachedTokensDetails +) -> CachedTokensDetails: + def _sum_optional(current_value: int | None, new_value: int | None) -> int | None: + if current_value is None and new_value is None: + return None + return (current_value or 0) + (new_value or 0) + + return CachedTokensDetails( + text_tokens=_sum_optional( + current.text_tokens if current is not None else None, new.text_tokens + ), + audio_tokens=_sum_optional( + current.audio_tokens if current is not None else None, new.audio_tokens + ), + image_tokens=_sum_optional( + current.image_tokens if current is not None else None, new.image_tokens + ), + ) + + +def _combine_prompt_tokens_details(combined: Usage, usage: Usage) -> None: + if not (hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details): + return + if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details: + combined.prompt_tokens_details = PromptTokensDetailsWrapper() + + # Check what keys exist in the model's prompt_tokens_details + # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings + for attr in _summable_prompt_token_fields(usage.prompt_tokens_details): + if ( + hasattr(usage.prompt_tokens_details, attr) + and not attr.startswith("_") + and not callable(_attribute_value(usage.prompt_tokens_details, attr)) + ): + current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0 + new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0 + if new_val is not None and isinstance(new_val, (int, float)): + setattr( + combined.prompt_tokens_details, + attr, + current_val + new_val, + ) + + new_cached_tokens_details: Final = getattr( + usage.prompt_tokens_details, "cached_tokens_details", None + ) + if isinstance(new_cached_tokens_details, CachedTokensDetails): + combined.prompt_tokens_details.cached_tokens_details = _combine_cached_tokens_details( + getattr(combined.prompt_tokens_details, "cached_tokens_details", None), + new_cached_tokens_details, + ) + + class BaseTokenUsageProcessor: @staticmethod def combine_usage_objects(usage_objects: list[Usage]) -> Usage: @@ -2318,7 +2373,6 @@ class BaseTokenUsageProcessor: """ from litellm.types.utils import ( CompletionTokensDetailsWrapper, - PromptTokensDetailsWrapper, Usage, ) @@ -2337,27 +2391,7 @@ class BaseTokenUsageProcessor: and isinstance(current_val, (int, float)) ): setattr(combined, attr, current_val + new_val) - # Handle nested prompt_tokens_details - if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: - if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details: - combined.prompt_tokens_details = PromptTokensDetailsWrapper() - - # Check what keys exist in the model's prompt_tokens_details - # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings - for attr in _summable_prompt_token_fields(usage.prompt_tokens_details): - if ( - hasattr(usage.prompt_tokens_details, attr) - and not attr.startswith("_") - and not callable(_attribute_value(usage.prompt_tokens_details, attr)) - ): - current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0 - new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0 - if new_val is not None and isinstance(new_val, (int, float)): - setattr( - combined.prompt_tokens_details, - attr, - current_val + new_val, - ) + _combine_prompt_tokens_details(combined, usage) # Handle nested completion_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e5977ca4156..18ef99597a0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -9,6 +9,8 @@ from types import MappingProxyType from typing import Any, Final, Literal, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from typing_extensions import ReadOnly + import litellm from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger @@ -772,6 +774,7 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int + cache_hit_audio_tokens: ReadOnly[int] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -802,12 +805,26 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: ) or None ) - text_tokens: Final = ( - cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None)) - or 0 # default to prompt tokens, if this field is not set + cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None) + cached_text_tokens: Final = _get_token_detail_value(cached_tokens_details, "text_tokens") or 0 + cached_audio_tokens: Final = _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0 + cached_image_tokens: Final = _get_token_detail_value(cached_tokens_details, "image_tokens") or 0 + text_tokens: Final = max( + ( + cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None)) + or 0 # default to prompt tokens, if this field is not set + ) + - cached_text_tokens, + 0, + ) + audio_tokens: Final = max( + (cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0) - cached_audio_tokens, + 0, + ) + image_tokens: Final = max( + (cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0) - cached_image_tokens, + 0, ) - audio_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0 - image_tokens: Final = cast(int | None, getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0 video_tokens: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0)) character_count: Final = ( cast( @@ -835,6 +852,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, + cache_hit_audio_tokens=min(cached_audio_tokens, cache_hit_tokens), cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -918,7 +936,16 @@ def _calculate_input_cost( prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost + cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] + audio_cache_read_rate: Final = _get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), + None, + ) + prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost + prompt_cost += float(cache_hit_audio_tokens) * ( + audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + ) ### AUDIO COST if prompt_tokens_details["audio_tokens"]: @@ -1149,6 +1176,7 @@ def generic_cost_per_token( ### PROCESSING COST prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, + cache_hit_audio_tokens=0, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1c0bd32d782..7f4ca991bd3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -32502,6 +32502,7 @@ }, "gpt-realtime": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, @@ -32535,6 +32536,7 @@ }, "gpt-realtime-1.5": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -32702,6 +32704,7 @@ }, "gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index fca5b0d11cf..cc84c68b0f5 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -45,6 +45,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import ) from litellm.types.llms.openai import ( AllMessageValues, + CachedTokensDetails, ChatCompletionImageObject, ChatCompletionImageUrlObject, ChatCompletionRedactedThinkingBlock, @@ -2681,27 +2682,31 @@ class LiteLLMCompletionResponsesConfig: # Translate prompt_tokens_details to input_tokens_details if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None: prompt_details: Final = usage.prompt_tokens_details - input_details_dict: Final[dict[str, int]] = {} - - if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None: - input_details_dict["cached_tokens"] = prompt_details.cached_tokens - else: - input_details_dict["cached_tokens"] = 0 - - if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None: - input_details_dict["text_tokens"] = prompt_details.text_tokens - - if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None: - input_details_dict["audio_tokens"] = prompt_details.audio_tokens - - cache_write_tokens = getattr(prompt_details, "cache_write_tokens", None) or getattr( - prompt_details, "cache_creation_tokens", None + cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None) + response_usage.input_tokens_details = InputTokensDetails( + cached_tokens=( + prompt_details.cached_tokens + if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None + else 0 + ), + text_tokens=( + prompt_details.text_tokens + if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None + else None + ), + audio_tokens=( + prompt_details.audio_tokens + if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None + else None + ), + cache_write_tokens=( + getattr(prompt_details, "cache_write_tokens", None) + or getattr(prompt_details, "cache_creation_tokens", None) + ), + cached_tokens_details=( + cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None + ), ) - if cache_write_tokens is not None: - input_details_dict["cache_write_tokens"] = cache_write_tokens - - if input_details_dict: - response_usage.input_tokens_details = InputTokensDetails(**input_details_dict) # Translate completion_tokens_details to output_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 599e978df6a..d63e3ddf0aa 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1179,6 +1179,9 @@ class ResponseAPILoggingUtils: audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), + cached_tokens_details=getattr( + response_api_usage.input_tokens_details, "cached_tokens_details", None + ), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index b7c4371f32f..274747b4193 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1284,9 +1284,16 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): model_config = {"extra": "allow"} +class CachedTokensDetails(BaseModel): + text_tokens: int | None = None + audio_tokens: int | None = None + image_tokens: int | None = None + + class InputTokensDetails(BaseLiteLLMOpenAIResponseObject): audio_tokens: int | None = None cached_tokens: int = 0 + cached_tokens_details: CachedTokensDetails | None = None text_tokens: int | None = None model_config = {"extra": "allow"} @@ -2204,10 +2211,17 @@ class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict): transcript: ReadOnly[str] +class OpenAIRealtimeCachedTokensDetails(TypedDict, total=False): + text_tokens: ReadOnly[int] + audio_tokens: ReadOnly[int] + image_tokens: ReadOnly[int] + + class OpenAIRealtimeUsageTokenDetails(TypedDict): audio_tokens: ReadOnly[int] text_tokens: ReadOnly[int] cached_tokens: NotRequired[ReadOnly[int]] + cached_tokens_details: NotRequired[ReadOnly[OpenAIRealtimeCachedTokensDetails]] class OpenAIRealtimeResponseUsage(TypedDict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ab0cc5f959c..39100031dcf 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -58,6 +58,7 @@ from .llms.base import HiddenParams from .llms.openai import ( AllMessageValues, Batch, + CachedTokensDetails, ChatCompletionAnnotation, ChatCompletionReasoningItem, ChatCompletionRedactedThinkingBlock, @@ -1707,6 +1708,9 @@ class PromptTokensDetailsWrapper( cache_creation_token_details: CacheCreationTokenDetails | None = None """Details of cache creation tokens sent to the model. Used for tracking 5m/1h cache creation tokens for Anthropic prompt caching.""" + cached_tokens_details: CachedTokensDetails | None = None + """Details of cached (cache-hit) tokens sent to the model. OpenAI realtime naming; carries the per-modality cache-read split.""" + def __setattr__(self, name: str, value: object) -> None: super().__setattr__(name, value) if name == "cache_write_tokens": @@ -1753,6 +1757,8 @@ class PromptTokensDetailsWrapper( del self.cache_creation_tokens if self.cache_creation_token_details is None: del self.cache_creation_token_details + if self.cached_tokens_details is None: + del self.cached_tokens_details class ServerToolUse(BaseModel): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1c0bd32d782..7f4ca991bd3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -32502,6 +32502,7 @@ }, "gpt-realtime": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, @@ -32535,6 +32536,7 @@ }, "gpt-realtime-1.5": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -32702,6 +32704,7 @@ }, "gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 3.2e-05, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index fbb9d178390..3709b526c3b 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2648,6 +2648,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, + "cache_hit_audio_tokens": 0, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, @@ -5147,3 +5148,70 @@ def test_generic_cost_per_token_bills_nested_reasoning_once_beside_audio_output( assert completion_cost == pytest.approx( 30 * info["output_cost_per_token"] + 70 * info["output_cost_per_audio_token"] ) + + +def test_cached_realtime_audio_tokens_billed_at_audio_cache_read_rate( + _local_model_cost_map: None, +) -> None: + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, + audio_tokens=167, + cached_tokens=192, + cached_tokens_details={"text_tokens": 64, "audio_tokens": 128}, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" + ) + assert prompt_cost == pytest.approx(0.0015328) + + +def test_prompt_tokens_details_without_cached_tokens_details_unchanged( + _local_model_cost_map: None, +) -> None: + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, audio_tokens=167, cached_tokens=192 + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" + ) + assert prompt_cost == pytest.approx(0.0029888) + + +def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: + model_info: ModelInfo = { + "input_cost_per_token": 4e-6, + "input_cost_per_audio_token": 32e-6, + "cache_read_input_token_cost": 5e-7, + } + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, + audio_tokens=167, + cached_tokens=192, + cached_tokens_details={"text_tokens": 64, "audio_tokens": 128}, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="some-realtime-model", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + ) + expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7 + assert prompt_cost == pytest.approx(expected) diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 9d9eefdceb3..4d06b5e7bdc 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -577,6 +577,47 @@ class TestResponseAPILoggingUtils: assert result.completion_tokens_details is not None assert result.completion_tokens_details.reasoning_tokens == 4 + def test_transform_realtime_usage_dict_keeps_cached_tokens_details(self): + usage = { + "input_tokens": 283, + "output_tokens": 0, + "total_tokens": 283, + "input_token_details": { + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + } + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens == 192 + assert result.prompt_tokens_details.cached_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64 + + def test_transform_response_api_usage_object_keeps_cached_tokens_details(self): + usage = ResponseAPIUsage( + input_tokens=283, + output_tokens=0, + total_tokens=283, + input_tokens_details={ + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + ) + + result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + + assert result.prompt_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens_details is not None + assert result.prompt_tokens_details.cached_tokens_details.audio_tokens == 128 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 64 + class TestResponsesAPIProviderSpecificParams: """ diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index f610821e06a..0b339e3d525 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4768,3 +4768,64 @@ def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() -> assert combined.completion_tokens_details.reasoning_tokens == 95 assert combined.completion_tokens_details.text_tokens == 38 assert combined.completion_tokens_details.audio_tokens == 0 + + +def test_realtime_combine_sums_nested_cached_tokens_details(): + results: OpenAIRealtimeStreamList = [ + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 283, + "output_tokens": 0, + "total_tokens": 283, + "input_token_details": { + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + } + }, + }, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 150, + "output_tokens": 0, + "total_tokens": 150, + "input_token_details": { + "text_tokens": 50, + "audio_tokens": 100, + "cached_tokens": 100, + "cached_tokens_details": {"audio_tokens": 100}, + }, + } + }, + }, + ] + + combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results, + ) + + assert combined.prompt_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens == 292 + assert combined.prompt_tokens_details.cached_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens_details.audio_tokens == 228 + assert combined.prompt_tokens_details.cached_tokens_details.text_tokens == 64 + assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None + + +def test_usage_without_cached_tokens_details_omits_key(): + usage = Usage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10), + ) + + dumped = usage.prompt_tokens_details.model_dump() + assert "cached_tokens_details" not in dumped + assert "cached_tokens_details" not in usage.prompt_tokens_details.model_dump_json() From 67fc9e4e3dcb94035c4b4d07d63c3565939d9a3f Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 10 Sep 2026 22:10:06 +0000 Subject: [PATCH 004/114] fix(responses): only emit cache_write_tokens when reported Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 16 +++------- .../transformation.py | 30 +++++++------------ .../test_litellm_completion_responses.py | 2 ++ 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7cd3ea8f303..8daa2de416b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2320,15 +2320,9 @@ def _combine_cached_tokens_details( return (current_value or 0) + (new_value or 0) return CachedTokensDetails( - text_tokens=_sum_optional( - current.text_tokens if current is not None else None, new.text_tokens - ), - audio_tokens=_sum_optional( - current.audio_tokens if current is not None else None, new.audio_tokens - ), - image_tokens=_sum_optional( - current.image_tokens if current is not None else None, new.image_tokens - ), + text_tokens=_sum_optional(current.text_tokens if current is not None else None, new.text_tokens), + audio_tokens=_sum_optional(current.audio_tokens if current is not None else None, new.audio_tokens), + image_tokens=_sum_optional(current.image_tokens if current is not None else None, new.image_tokens), ) @@ -2355,9 +2349,7 @@ def _combine_prompt_tokens_details(combined: Usage, usage: Usage) -> None: current_val + new_val, ) - new_cached_tokens_details: Final = getattr( - usage.prompt_tokens_details, "cached_tokens_details", None - ) + new_cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None) if isinstance(new_cached_tokens_details, CachedTokensDetails): combined.prompt_tokens_details.cached_tokens_details = _combine_cached_tokens_details( getattr(combined.prompt_tokens_details, "cached_tokens_details", None), diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index cc84c68b0f5..e6f90b99b60 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2683,30 +2683,20 @@ class LiteLLMCompletionResponsesConfig: if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details is not None: prompt_details: Final = usage.prompt_tokens_details cached_tokens_details: Final = getattr(prompt_details, "cached_tokens_details", None) - response_usage.input_tokens_details = InputTokensDetails( - cached_tokens=( - prompt_details.cached_tokens - if hasattr(prompt_details, "cached_tokens") and prompt_details.cached_tokens is not None - else 0 - ), - text_tokens=( - prompt_details.text_tokens - if hasattr(prompt_details, "text_tokens") and prompt_details.text_tokens is not None - else None - ), - audio_tokens=( - prompt_details.audio_tokens - if hasattr(prompt_details, "audio_tokens") and prompt_details.audio_tokens is not None - else None - ), - cache_write_tokens=( - getattr(prompt_details, "cache_write_tokens", None) - or getattr(prompt_details, "cache_creation_tokens", None) - ), + cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr( + prompt_details, "cache_creation_tokens", None + ) + input_tokens_details: Final = InputTokensDetails( + cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, + text_tokens=prompt_details.text_tokens, + audio_tokens=prompt_details.audio_tokens, cached_tokens_details=( cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None ), ) + if cache_write_tokens is not None: + setattr(input_tokens_details, "cache_write_tokens", cache_write_tokens) + response_usage.input_tokens_details = input_tokens_details # Translate completion_tokens_details to output_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None: diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 46249e50572..be96c2a4bf5 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -2605,6 +2605,7 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 5 assert response_usage.input_tokens_details.text_tokens == 8 + assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump() def test_transform_usage_with_cached_tokens_gemini(self): """Test that cached_tokens from Gemini are properly transformed to input_tokens_details""" @@ -2667,6 +2668,7 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 100 assert getattr(response_usage.input_tokens_details, "cache_write_tokens", None) == 800 + assert response_usage.input_tokens_details.model_dump()["cache_write_tokens"] == 800 def test_transform_usage_with_reasoning_tokens_gemini(self): """Test that reasoning_tokens from Gemini are properly transformed to output_tokens_details""" From 5737cab258b405689a55e1e6dcaec385673fa572 Mon Sep 17 00:00:00 2001 From: shivam Date: Thu, 10 Sep 2026 22:27:58 +0000 Subject: [PATCH 005/114] fix(cost): cap nested cached modality counts at cached_tokens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 2 -- .../litellm_core_utils/llm_cost_calc/utils.py | 16 +++++++++++---- .../llm_cost_calc/test_llm_cost_calc_utils.py | 20 +++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8daa2de416b..b865318f3af 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2332,8 +2332,6 @@ def _combine_prompt_tokens_details(combined: Usage, usage: Usage) -> None: if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details: combined.prompt_tokens_details = PromptTokensDetailsWrapper() - # Check what keys exist in the model's prompt_tokens_details - # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings for attr in _summable_prompt_token_fields(usage.prompt_tokens_details): if ( hasattr(usage.prompt_tokens_details, attr) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 18ef99597a0..dc689ca9618 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -806,9 +806,17 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: or None ) cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None) - cached_text_tokens: Final = _get_token_detail_value(cached_tokens_details, "text_tokens") or 0 - cached_audio_tokens: Final = _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0 - cached_image_tokens: Final = _get_token_detail_value(cached_tokens_details, "image_tokens") or 0 + cached_audio_tokens: Final = min( + _get_token_detail_value(cached_tokens_details, "audio_tokens") or 0, cache_hit_tokens + ) + cached_text_tokens: Final = min( + _get_token_detail_value(cached_tokens_details, "text_tokens") or 0, + cache_hit_tokens - cached_audio_tokens, + ) + cached_image_tokens: Final = min( + _get_token_detail_value(cached_tokens_details, "image_tokens") or 0, + cache_hit_tokens - cached_audio_tokens - cached_text_tokens, + ) text_tokens: Final = max( ( cast(int | None, getattr(usage.prompt_tokens_details, "text_tokens", None)) @@ -852,7 +860,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, - cache_hit_audio_tokens=min(cached_audio_tokens, cache_hit_tokens), + cache_hit_audio_tokens=cached_audio_tokens, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 3709b526c3b..33825c8dd01 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -5215,3 +5215,23 @@ def test_cached_audio_tokens_fall_back_to_cache_read_input_token_cost() -> None: ) expected = 52 * 4e-6 + 64 * 5e-7 + 39 * 32e-6 + 128 * 5e-7 assert prompt_cost == pytest.approx(expected) + + +def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None) -> None: + """Nested cached_tokens_details exceeding cached_tokens must not over-subtract the audio bucket.""" + usage = Usage( + prompt_tokens=283, + completion_tokens=0, + total_tokens=283, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=116, + audio_tokens=167, + cached_tokens=100, + cached_tokens_details={"audio_tokens": 128}, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" + ) + assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7) From a067557dae5c0bb52fdb11176437a39c9a0d9ac3 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 11 Sep 2026 23:16:53 +0000 Subject: [PATCH 006/114] fix(otel): index the opener and the latest prompt turns, not the oldest A value length limit clips the input.value blob, so the per-index keys are the only untruncated copy of a message. Indexing the leading prompt messages dropped the live user turn from every span attribute on long conversations. Keep message 0 and the most recent turns under the same span-wide budget, original indices preserved, reply reservation unchanged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../otel/mappers/openinference.py | 31 +++++++++++++------ .../integrations/otel/test_otel_v2_emitter.py | 29 ++++++++++++++++- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/otel/mappers/openinference.py b/litellm/integrations/otel/mappers/openinference.py index 1e2dbf6974d..1fa19b8d4a5 100644 --- a/litellm/integrations/otel/mappers/openinference.py +++ b/litellm/integrations/otel/mappers/openinference.py @@ -92,8 +92,13 @@ class OpenInferenceMapper: return { **collect(self._LLM_CALL_ATTRS, data), **collect(self._BLOB_ATTRS, data), - **self._messages("llm.input_messages", "input.value", data.messages_in, indexed_in), - **self._messages("llm.output_messages", "output.value", outputs, indexed_out), + **self._messages( + "llm.input_messages", + "input.value", + data.messages_in, + self._prompt_positions(len(data.messages_in), indexed_in), + ), + **self._messages("llm.output_messages", "output.value", outputs, range(indexed_out)), **self._tools(data), } @@ -109,18 +114,26 @@ class OpenInferenceMapper: return _MAX_INDEXED_MESSAGES - indexed_out, indexed_out @staticmethod - def _messages(prefix: str, value_key: str, messages: Sequence[object], indexed: int) -> AttributeMap: - """``{prefix}.{idx}.message.*`` keys for the leading ``indexed`` messages + the ``value_key`` blob of all.""" + def _prompt_positions(total: int, indexed: int) -> tuple[int, ...]: + """Which prompt messages get per-index attributes: message 0 and the most recent turns. + + A value length limit clips the ``input.value`` blob, so the system prompt and the + live turn each keep a short key of their own. The middle of a long prompt does not. + """ + if total <= indexed: + return tuple(range(total)) + return (0, *range(total - indexed + 1, total)) + + @staticmethod + def _messages(prefix: str, value_key: str, messages: Sequence[object], positions: Sequence[int]) -> AttributeMap: + """``{prefix}.{idx}.message.*`` keys for the messages at ``positions`` + the ``value_key`` blob of all.""" parsed: Final = [(m.get("role") if isinstance(m, dict) else None, message_content(m)) for m in messages] attrs: Final = drop_none( { key: value - for idx, (role, content) in enumerate(parsed[:indexed]) + for idx, (role, content) in ((idx, parsed[idx]) for idx in positions) for key, value in ( - ( - f"{prefix}.{idx}.message.role", - role if isinstance(role, str) else None, - ), + (f"{prefix}.{idx}.message.role", role if isinstance(role, str) else None), (f"{prefix}.{idx}.message.content", content), ) } diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py index f571ab7004b..6491ab0f79f 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py @@ -505,7 +505,8 @@ def test_long_conversation_does_not_evict_core_attributes(turns): assert a["llm.input_messages.0.message.content"] == "turn 0" assert a["llm.output_messages.0.message.content"] == "reply 0" - assert f"llm.input_messages.{turns - 1}.message.role" not in a + assert a[f"llm.input_messages.{turns - 1}.message.content"] == f"turn {turns - 1}" + assert f"llm.input_messages.{turns // 2}.message.role" not in a assert len(json.loads(a["input.value"])) == turns assert len(json.loads(a["output.value"])) == 1 assert len(json.loads(a[GenAI.INPUT_MESSAGES])) == turns @@ -520,6 +521,31 @@ def test_short_conversation_keeps_every_message_indexed(): assert a[f"llm.output_messages.{idx}.message.content"] == f"reply {idx}" +def test_indexed_prompt_keeps_opener_and_latest_turns_under_a_value_length_limit(monkeypatch): + """The per-index keys are the only untruncated copy once the SDK clips string values. + + Operators bound attribute sizes with ``OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT``, + which cuts the ``input.value`` blob short. The system prompt and the live turn + then have to survive as their own short keys, whatever the conversation length. + """ + monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", "256") + payload = _conversation_payload(60) + payload["messages"][0] = {"role": "system", "content": "be terse"} + payload["messages"][-1] = {"role": "user", "content": "LATEST-TURN"} + a = _conversation_span(["genai", "openinference"], payload).attributes + + assert len(a["input.value"]) == 256 + assert a["llm.input_messages.0.message.role"] == "system" + assert a["llm.input_messages.0.message.content"] == "be terse" + assert a["llm.input_messages.59.message.role"] == "user" + assert a["llm.input_messages.59.message.content"] == "LATEST-TURN" + assert a["llm.output_messages.0.message.content"] == "reply 0" + assert [int(key.split(".")[2]) for key in a if key.endswith("message.content") and key.startswith("llm.input_")] == [ + 0, + *range(54, 60), + ] + + def test_message_cap_is_shared_across_input_and_output(): """One span-wide allowance covers both directions, and the response always keeps a share. @@ -588,4 +614,5 @@ def test_fully_populated_span_with_every_vocabulary_stays_within_the_attribute_l assert a[f"{LiteLLM.COST_PREFIX}total"] == 0.002 assert a[LiteLLM.TOOLS_DECLARED] == 127 assert a["llm.input_messages.0.message.content"] == "turn 0" + assert a["llm.input_messages.199.message.content"] == "turn 199" assert a["llm.output_messages.0.message.content"] == "reply 0" From a15309dfe820836a41e914228359d7b5becc3744 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 11 Sep 2026 23:32:44 +0000 Subject: [PATCH 007/114] refactor(otel): trim the message cap docstrings to one line each Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../otel/mappers/openinference.py | 13 ++------- litellm/integrations/otel/mappers/utils.py | 13 ++------- .../integrations/otel/test_otel_v2_emitter.py | 29 +++---------------- 3 files changed, 9 insertions(+), 46 deletions(-) diff --git a/litellm/integrations/otel/mappers/openinference.py b/litellm/integrations/otel/mappers/openinference.py index 1fa19b8d4a5..a7e0f1af3ac 100644 --- a/litellm/integrations/otel/mappers/openinference.py +++ b/litellm/integrations/otel/mappers/openinference.py @@ -104,22 +104,13 @@ class OpenInferenceMapper: @staticmethod def _indexed_split(inputs: int, outputs: int) -> tuple[int, int]: - """How many prompt and response messages get per-index attributes. - - Both directions share one span-wide allowance. The response is reserved at - least half of it, so a long prompt can never push the completion off the - span, and the prompt takes whatever the response leaves unused. - """ + """Prompt and response share one allowance; the response is reserved at least half of it.""" indexed_out: Final = min(outputs, max(_MAX_INDEXED_MESSAGES // 2, _MAX_INDEXED_MESSAGES - inputs)) return _MAX_INDEXED_MESSAGES - indexed_out, indexed_out @staticmethod def _prompt_positions(total: int, indexed: int) -> tuple[int, ...]: - """Which prompt messages get per-index attributes: message 0 and the most recent turns. - - A value length limit clips the ``input.value`` blob, so the system prompt and the - live turn each keep a short key of their own. The middle of a long prompt does not. - """ + """Prompt messages that get per-index attributes: message 0 and the most recent turns.""" if total <= indexed: return tuple(range(total)) return (0, *range(total - indexed + 1, total)) diff --git a/litellm/integrations/otel/mappers/utils.py b/litellm/integrations/otel/mappers/utils.py index d8918491720..c023621d2ef 100644 --- a/litellm/integrations/otel/mappers/utils.py +++ b/litellm/integrations/otel/mappers/utils.py @@ -33,17 +33,10 @@ core telemetry no matter how many vocabularies are configured. MAX_MESSAGE_ATTRS_PER_SPAN: Final = DEFAULT_SPAN_ATTRIBUTE_LIMIT // 8 -"""Span-wide ceiling on attributes spent spelling out chat messages per index. +"""Span-wide ceiling on per-index chat message attributes, prompt and response together. -A conversation is the other unbounded family: two attributes per message, for -the prompt and the response alike, on the same span. Past a few dozen turns the -family alone exceeds the span attribute limit and evicts the core telemetry -written before it. The ceiling covers both directions together, since a budget -handed to each direction separately doubles. An eighth is the largest share -that still fits beside the tool ceiling and the core of every vocabulary at -once, request parameters, cost breakdown and identity included. The complete -conversation still rides the JSON blob attributes; only the per-index -convenience keys are capped. +An eighth is the largest share that still fits beside the tool ceiling and the core +of every vocabulary at once. The complete conversation still rides the JSON blobs. """ diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py index 6491ab0f79f..16fbb242ebd 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_emitter.py @@ -485,13 +485,7 @@ def _indexed_message_count(attributes, prefix): @pytest.mark.parametrize("turns", [60, 200]) def test_long_conversation_does_not_evict_core_attributes(turns): - """Per-message OpenInference attributes must never crowd core telemetry off the span. - - With content capture on, the OpenInference vocabulary spells every prompt and - response message out as two per-index attributes. A few dozen turns overruns - the OTel SDK's 128-attribute span limit, which evicts oldest-first, so the - ``gen_ai.*`` set written before it is what disappears. - """ + """Per-message OpenInference attributes must never crowd core telemetry off the span.""" span = _conversation_span(["genai", "openinference"], _conversation_payload(turns)) a = span.attributes @@ -522,12 +516,7 @@ def test_short_conversation_keeps_every_message_indexed(): def test_indexed_prompt_keeps_opener_and_latest_turns_under_a_value_length_limit(monkeypatch): - """The per-index keys are the only untruncated copy once the SDK clips string values. - - Operators bound attribute sizes with ``OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT``, - which cuts the ``input.value`` blob short. The system prompt and the live turn - then have to survive as their own short keys, whatever the conversation length. - """ + """The system prompt and the live turn keep their own keys once the SDK clips ``input.value``.""" monkeypatch.setenv("OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", "256") payload = _conversation_payload(60) payload["messages"][0] = {"role": "system", "content": "be terse"} @@ -547,11 +536,7 @@ def test_indexed_prompt_keeps_opener_and_latest_turns_under_a_value_length_limit def test_message_cap_is_shared_across_input_and_output(): - """One span-wide allowance covers both directions, and the response always keeps a share. - - A long prompt takes what a single reply leaves over, and a many-choice reply - cannot take the whole allowance away from the prompt either. - """ + """One span-wide allowance covers both directions, and the response always keeps a share.""" long_prompt = _conversation_span(["genai", "openinference"], _conversation_payload(60, choices=1)).attributes many_choices = _conversation_span(["genai", "openinference"], _conversation_payload(60, choices=20)).attributes @@ -569,13 +554,7 @@ def test_message_cap_is_shared_across_input_and_output(): def test_fully_populated_span_with_every_vocabulary_stays_within_the_attribute_limit(): - """Every capped family maxed at once still leaves the whole core intact. - - Every vocabulary in the registry plus ``legacy``, every request parameter, - every cost component, a hundred-plus tools, a two-hundred-turn prompt and - twenty choices is the worst case the two span-wide ceilings have to absorb - together. - """ + """Every capped family maxed at once still leaves the whole core intact.""" payload = _conversation_payload( 200, choices=20, From 107b4ec4db64985de0b3651f401b290ea09e81ed Mon Sep 17 00:00:00 2001 From: yassin Date: Sat, 12 Sep 2026 01:12:39 +0000 Subject: [PATCH 008/114] fix(redis): log a timeout streak once per interval instead of one line per cache call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 140 ++++++++++++------ litellm/constants.py | 3 + tests/test_litellm/caching/test_dual_cache.py | 57 +++++++ .../test_litellm/caching/test_redis_cache.py | 57 +++++++ 4 files changed, 211 insertions(+), 46 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 2c36995c4f8..eaac7ef7b0b 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -15,6 +15,7 @@ import hashlib import inspect import json import logging +import threading import time from collections.abc import Awaitable, Callable, Sequence from contextvars import ContextVar @@ -32,6 +33,7 @@ from litellm.constants import ( REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD, REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT, REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION, + REDIS_TIMEOUT_LOG_INTERVAL, ) from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs from litellm.litellm_core_utils.coroutine_checker import coroutine_checker @@ -404,13 +406,58 @@ class RedisCircuitBreakerOpenError(Exception): pass +class _RedisTimeoutLogThrottle: + """Admits one Redis timeout log line per interval and counts the timeouts it suppressed in between.""" + + def __init__(self, interval: float, clock: Callable[[], float] = time.time) -> None: + self.interval = interval + self._clock = clock + self._lock = threading.Lock() + self._last_logged_at: float | None = None + self._suppressed = 0 + + def admit(self) -> int | None: + """Return the number of timeouts suppressed since the last admitted line, or None to suppress this one.""" + with self._lock: + now: Final = self._clock() + if self._last_logged_at is not None and now - self._last_logged_at < self.interval: + self._suppressed += 1 + return None + suppressed: Final = self._suppressed + self._suppressed = 0 + self._last_logged_at = now + return suppressed + + +_redis_timeout_log_throttle: Final = _RedisTimeoutLogThrottle(REDIS_TIMEOUT_LOG_INTERVAL) + + def log_redis_failure( logger: logging.Logger, level: int, message: str, exc: BaseException, with_traceback: bool = False ) -> None: if isinstance(exc, RedisCircuitBreakerOpenError): - logger.debug("%s: %s", message, exc) + logger.debug("%s: %s", message, exc, stacklevel=2) return - logger.log(level, "%s: %s", message, exc, exc_info=exc if with_traceback else None) + exc_info: Final = exc if with_traceback else None + if not _is_redis_timeout_failure(exc): + logger.log(level, "%s: %s", message, exc, exc_info=exc_info, stacklevel=2) + return + suppressed: Final = _redis_timeout_log_throttle.admit() + if suppressed is None: + logger.debug("%s: %s", message, exc, stacklevel=2) + return + if suppressed == 0: + logger.log(level, "%s: %s", message, exc, exc_info=exc_info, stacklevel=2) + return + logger.log( + level, + "%s: %s (%d more Redis timeouts since the previous Redis timeout line were logged at DEBUG)", + message, + exc, + suppressed, + exc_info=exc_info, + stacklevel=2, + ) @dataclass(frozen=True, slots=True) @@ -783,10 +830,8 @@ class RedisCache(BaseCache): ## LOGGING ## end_time = time.time() _duration = end_time - start_time - verbose_logger.error( - "LiteLLM Redis Caching: increment_cache() - Got exception from REDIS %s, Writing value=%s", - str(e), - value, + log_redis_failure( + verbose_logger, logging.ERROR, "LiteLLM Redis Caching: increment_cache() - Got exception from REDIS", e ) raise e @@ -992,11 +1037,8 @@ class RedisCache(BaseCache): call_type=f"async_set_cache <- {_get_call_stack_info()}", ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r", - str(e), - key, - value, + log_redis_failure( + verbose_logger, logging.ERROR, "LiteLLM Redis Caching: async set() - Got exception from REDIS", e ) raise e @@ -1044,10 +1086,8 @@ class RedisCache(BaseCache): event_metadata={"key": key}, ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s", - str(e), - value, + log_redis_failure( + verbose_logger, logging.ERROR, "LiteLLM Redis Caching: async set() - Got exception from REDIS", e ) _record_swallowed_redis_failure(self._circuit_breaker, e) @@ -1094,7 +1134,6 @@ class RedisCache(BaseCache): start_time: Final = time.time() print_verbose(f"Set Async Redis Cache: key list: {cache_list}\nttl={ttl}, redis_version={self.redis_version}") - cache_value: Final = None try: async with _redis_client.pipeline(transaction=False) as pipe: results: Final = await self._pipeline_helper(pipe, cache_list, ttl) @@ -1131,10 +1170,11 @@ class RedisCache(BaseCache): ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS %s, Writing value=%s", - str(e), - cache_value, + log_redis_failure( + verbose_logger, + logging.ERROR, + "LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS", + e, ) _record_swallowed_redis_failure(self._circuit_breaker, e) @@ -1177,10 +1217,8 @@ class RedisCache(BaseCache): ) ) # NON blocking - notify users Redis is throwing an exception - verbose_logger.error( - "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s", - str(e), - value, + log_redis_failure( + verbose_logger, logging.ERROR, "LiteLLM Redis Caching: async set() - Got exception from REDIS", e ) raise e @@ -1216,10 +1254,11 @@ class RedisCache(BaseCache): ) ) # NON blocking - notify users Redis is throwing an exception - verbose_logger.error( - "LiteLLM Redis Caching: async set_cache_sadd() - Got exception from REDIS %s, Writing value=%s", - str(e), - value, + log_redis_failure( + verbose_logger, + logging.ERROR, + "LiteLLM Redis Caching: async set_cache_sadd() - Got exception from REDIS", + e, ) _record_swallowed_redis_failure(self._circuit_breaker, e) @@ -1288,10 +1327,11 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async async_increment() - Got exception from REDIS %s, Writing value=%s", - str(e), - value, + log_redis_failure( + verbose_logger, + logging.ERROR, + "LiteLLM Redis Caching: async async_increment() - Got exception from REDIS", + e, ) raise e @@ -1377,7 +1417,9 @@ class RedisCache(BaseCache): print_verbose(f"Got Redis Cache: key: {key}, cached_response {cached_response}") return self._get_cache_logic(cached_response=cached_response) except Exception as e: - verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: %s", e) + log_redis_failure( + verbose_logger, logging.ERROR, "litellm.caching.caching: get() - Got exception from REDIS", e + ) _record_swallowed_redis_failure(self._circuit_breaker, e) def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]: @@ -1455,7 +1497,7 @@ class RedisCache(BaseCache): end_time=failed_at, parent_otel_span=parent_otel_span, ) - verbose_logger.error("Error occurred in batch get cache - %s", e) + log_redis_failure(verbose_logger, logging.ERROR, "Error occurred in batch get cache", e) _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict @@ -1574,7 +1616,7 @@ class RedisCache(BaseCache): parent_otel_span=parent_otel_span, ) ) - verbose_logger.error("Error occurred in async batch get cache - %s", e) + log_redis_failure(verbose_logger, logging.ERROR, "Error occurred in async batch get cache", e) _record_swallowed_redis_failure(self._circuit_breaker, e) return key_value_dict @@ -1799,9 +1841,11 @@ class RedisCache(BaseCache): parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs), ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async increment_pipeline() - Got exception from REDIS %s", - str(e), + log_redis_failure( + verbose_logger, + logging.ERROR, + "LiteLLM Redis Caching: async increment_pipeline() - Got exception from REDIS", + e, ) raise e @@ -1878,7 +1922,7 @@ class RedisCache(BaseCache): call_type=f"async_rpush <- {_get_call_stack_info()}", ) ) - verbose_logger.error("LiteLLM Redis Cache RPUSH: - Got exception from REDIS : %s", e) + log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Redis Cache RPUSH: - Got exception from REDIS", e) raise e async def _pipeline_rpush_helper( @@ -1946,9 +1990,11 @@ class RedisCache(BaseCache): call_type=f"async_rpush_pipeline <- {_get_call_stack_info()}", ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async_rpush_pipeline() - Got exception from REDIS %s", - str(e), + log_redis_failure( + verbose_logger, + logging.ERROR, + "LiteLLM Redis Caching: async_rpush_pipeline() - Got exception from REDIS", + e, ) raise e @@ -2024,7 +2070,7 @@ class RedisCache(BaseCache): call_type=f"async_lpop <- {_get_call_stack_info()}", ) ) - verbose_logger.error("LiteLLM Redis Cache LPOP: - Got exception from REDIS : %s", e) + log_redis_failure(verbose_logger, logging.ERROR, "LiteLLM Redis Cache LPOP: - Got exception from REDIS", e) raise e async def _pipeline_lpop_helper( @@ -2135,8 +2181,10 @@ class RedisCache(BaseCache): call_type=f"async_lpop_pipeline <- {_get_call_stack_info()}", ) ) - verbose_logger.error( - "LiteLLM Redis Caching: async_lpop_pipeline() - Got exception from REDIS %s", - str(e), + log_redis_failure( + verbose_logger, + logging.ERROR, + "LiteLLM Redis Caching: async_lpop_pipeline() - Got exception from REDIS", + e, ) raise e diff --git a/litellm/constants.py b/litellm/constants.py index 6b984c2673c..a32551b4480 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -459,6 +459,9 @@ REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED" # minimum seconds a timeout-only failure streak must span before it can open the breaker, # so one event-loop stall timing out many queued calls at once does not trip it REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0)) +# seconds between Redis timeout log lines: the first timeout of a streak logs at the caller's level, +# later ones log at DEBUG until the interval passes and one line summarizes how many were suppressed +REDIS_TIMEOUT_LOG_INTERVAL: Final = float(os.getenv("REDIS_TIMEOUT_LOG_INTERVAL", "5.0")) # Seconds of idle before a Redis cluster connection is validated with a PING and # reconnected if dead, so a connection silently dropped by a cluster restart # (e.g. ElastiCache Serverless maintenance) is not reused while broken diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 4c9068722b8..850fa14106b 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -704,3 +704,60 @@ async def test_open_breaker_keeps_async_batch_read_memory_hits_and_releases_rese assert list(await cache.async_batch_get_cache(["k1", "k2"])) == ["v1", None] assert "k2" not in cache.last_redis_batch_access_time + + +@pytest.mark.asyncio +async def test_redis_timeouts_falling_back_to_memory_log_once_per_interval(caplog, monkeypatch): + """The in-memory fallback WARNING must not repeat for every timed-out increment during a blip. + + The rate limiter's pipeline increments and the dual cache increments each logged a WARNING per + call while Redis timed out, hundreds of lines per second before the breaker opened. The first + timeout of a streak keeps its WARNING, the rest are DEBUG until the summary interval passes. + """ + from redis.exceptions import TimeoutError as RedisTimeoutError + + from litellm.caching import redis_cache as redis_cache_module + from litellm.caching.redis_cache import _RedisTimeoutLogThrottle + + clock = MagicMock(return_value=1_000.0) + monkeypatch.setattr( + redis_cache_module, "_redis_timeout_log_throttle", _RedisTimeoutLogThrottle(interval=5.0, clock=clock) + ) + + class _TimingOutRedis: + async def async_increment_pipeline(self, increment_list, **kwargs): + raise RedisTimeoutError("Timeout reading from 127.0.0.1:6379") + + async def async_increment(self, key, value, **kwargs): + raise RedisTimeoutError("Timeout reading from 127.0.0.1:6379") + + cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=_TimingOutRedis()) # pyright: ignore[reportArgumentType] # duck-typed Redis double + increments = [RedisPipelineIncrementOperation(key="k", increment_value=1.0, ttl=60)] + + with caplog.at_level(logging.DEBUG, logger="LiteLLM"): + for _ in range(100): + await cache.async_increment_cache_pipeline(increment_list=increments) + await cache.async_increment_cache("k", 1.0) + + visible = [r for r in caplog.records if r.levelno >= logging.WARNING] + assert [(r.levelno, r.getMessage()) for r in visible] == [ + ( + logging.WARNING, + "Redis async_increment_cache_pipeline failed, falling back to in-memory result:" + " Timeout reading from 127.0.0.1:6379", + ) + ] + assert visible[0].filename == "dual_cache.py" + assert sum("Timeout reading from" in r.getMessage() for r in caplog.records) == 200 + + caplog.clear() + clock.return_value += 5.0 + with caplog.at_level(logging.DEBUG, logger="LiteLLM"): + await cache.async_increment_cache("k", 1.0) + assert [(r.levelno, r.getMessage()) for r in caplog.records] == [ + ( + logging.WARNING, + "Redis async_increment_cache failed, falling back to in-memory result: Timeout reading from 127.0.0.1:6379" + " (199 more Redis timeouts since the previous Redis timeout line were logged at DEBUG)", + ) + ] diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index bcae33b976e..d0974b2420c 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1202,3 +1202,60 @@ async def test_a_probe_overtaken_by_a_later_outage_leaves_the_breaker_to_the_new new_probe_release.set() assert await new_probe == "new probe" assert breaker._state == breaker.CLOSED + + +def test_timeouts_during_a_blip_log_once_per_interval_not_once_per_call(sync_batch_redis_cache, caplog, monkeypatch): + """A Redis latency blip must not write one ERROR line per timed-out cache call. + + Before the breaker opens (up to REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION of timeouts) every + cache operation logged its own ERROR or WARNING line, so one single-worker proxy wrote + ~1100 lines in 5 s at LITELLM_LOG=WARNING. A timeout streak now logs its first failure, then + one summary line per REDIS_TIMEOUT_LOG_INTERVAL carrying the count of suppressed timeouts, + while every timeout stays visible at DEBUG. Hard connectivity failures keep their per-call line. + """ + import logging + + from redis.exceptions import TimeoutError as RedisTimeoutError + + from litellm.caching import redis_cache as redis_cache_module + from litellm.caching.redis_cache import _RedisTimeoutLogThrottle + + clock = MagicMock(return_value=1_000.0) + monkeypatch.setattr( + redis_cache_module, "_redis_timeout_log_throttle", _RedisTimeoutLogThrottle(interval=5.0, clock=clock) + ) + sync_batch_redis_cache.redis_client.get.side_effect = RedisTimeoutError("Timeout reading from 127.0.0.1:6379") + sync_batch_redis_cache.redis_client.mget.side_effect = RedisTimeoutError("Timeout reading from 127.0.0.1:6379") + + with caplog.at_level(logging.DEBUG, logger="LiteLLM"): + for _ in range(200): + assert sync_batch_redis_cache.get_cache("lit7520") is None + assert sync_batch_redis_cache.batch_get_cache(key_list=["lit7520"]) == {} + + timeout_records = [r for r in caplog.records if "Timeout reading from" in r.getMessage()] + assert len(timeout_records) == 201, "every timeout must stay visible at DEBUG" + assert [r.getMessage() for r in timeout_records if r.levelno >= logging.WARNING] == [ + "litellm.caching.caching: get() - Got exception from REDIS: Timeout reading from 127.0.0.1:6379" + ] + assert timeout_records[0].levelno == logging.ERROR + assert timeout_records[0].filename == "redis_cache.py" + assert timeout_records[0].lineno != timeout_records[-1].lineno, "the record must point at the cache operation" + + caplog.clear() + clock.return_value += 5.0 + with caplog.at_level(logging.DEBUG, logger="LiteLLM"): + assert sync_batch_redis_cache.batch_get_cache(key_list=["lit7520"]) == {} + assert [(r.levelno, r.getMessage()) for r in caplog.records] == [ + ( + logging.ERROR, + "Error occurred in batch get cache: Timeout reading from 127.0.0.1:6379" + " (200 more Redis timeouts since the previous Redis timeout line were logged at DEBUG)", + ) + ] + + caplog.clear() + sync_batch_redis_cache.redis_client.get.side_effect = OSError("redis unavailable") + with caplog.at_level(logging.DEBUG, logger="LiteLLM"): + for _ in range(3): + assert sync_batch_redis_cache.get_cache("lit7520") is None + assert [r.levelno for r in caplog.records if "redis unavailable" in r.getMessage()] == [logging.ERROR] * 3 From 9c84e98fb22bd0f6e2c359f335bbc329181bb8bd Mon Sep 17 00:00:00 2001 From: yassin Date: Sat, 12 Sep 2026 01:25:32 +0000 Subject: [PATCH 009/114] fix(proxy): treat a Redis timeout in spend counter increments as an already-logged cache failure The cost tracking callback logged its own ERROR with a traceback for every request whose spend counter increment timed out, on top of the cache layer's throttled line. Timeouts now take the same path as breaker-open refusals: invalidate the counters and return. Also exposes is_redis_timeout_failure publicly for that caller and drops the comment on the new constant Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 10 +++---- litellm/constants.py | 2 -- litellm/proxy/proxy_server.py | 4 +-- .../test_litellm/caching/test_redis_cache.py | 26 +++++++++---------- .../proxy/proxy_server/test_spend_counters.py | 22 ++++++++++++++++ 5 files changed, 42 insertions(+), 22 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index eaac7ef7b0b..7a3e689a667 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -330,7 +330,7 @@ def _redis_timeout_error_types() -> tuple[type, ...]: return (RedisTimeoutError, TimeoutError) -def _is_redis_timeout_failure(exc: BaseException) -> bool: +def is_redis_timeout_failure(exc: BaseException) -> bool: return isinstance(exc, _redis_timeout_error_types()) @@ -398,7 +398,7 @@ def _record_swallowed_redis_failure(breaker: RedisCircuitBreaker, exc: BaseExcep """ if not _is_redis_health_failure(exc): return - breaker.record_failure(is_timeout=_is_redis_timeout_failure(exc)) + breaker.record_failure(is_timeout=is_redis_timeout_failure(exc)) _swallowed_redis_failures.set(_swallowed_redis_failures.get() + 1) @@ -439,7 +439,7 @@ def log_redis_failure( logger.debug("%s: %s", message, exc, stacklevel=2) return exc_info: Final = exc if with_traceback else None - if not _is_redis_timeout_failure(exc): + if not is_redis_timeout_failure(exc): logger.log(level, "%s: %s", message, exc, exc_info=exc_info, stacklevel=2) return suppressed: Final = _redis_timeout_log_throttle.admit() @@ -504,7 +504,7 @@ async def _run_under_circuit_breaker( result: Final = await call() except Exception as e: if _is_redis_health_failure(e): - breaker.record_failure(is_timeout=_is_redis_timeout_failure(e)) + breaker.record_failure(is_timeout=is_redis_timeout_failure(e)) raise _exit_circuit_breaker(breaker, admission) return result @@ -521,7 +521,7 @@ def _run_under_circuit_breaker_sync( result: Final = call() except Exception as e: if _is_redis_health_failure(e): - breaker.record_failure(is_timeout=_is_redis_timeout_failure(e)) + breaker.record_failure(is_timeout=is_redis_timeout_failure(e)) raise _exit_circuit_breaker(breaker, admission) return result diff --git a/litellm/constants.py b/litellm/constants.py index a32551b4480..60e1c682238 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -459,8 +459,6 @@ REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED" # minimum seconds a timeout-only failure streak must span before it can open the breaker, # so one event-loop stall timing out many queued calls at once does not trip it REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0)) -# seconds between Redis timeout log lines: the first timeout of a streak logs at the caller's level, -# later ones log at DEBUG until the interval passes and one line summarizes how many were suppressed REDIS_TIMEOUT_LOG_INTERVAL: Final = float(os.getenv("REDIS_TIMEOUT_LOG_INTERVAL", "5.0")) # Seconds of idle before a Redis cluster connection is validated with a PING and # reconnected if dead, so a connection silently dropped by a cluster restart diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c8fd7edfed6..318ea96dbcb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -251,7 +251,7 @@ import litellm._redis from litellm import Router from litellm._logging import _redact_string, verbose_proxy_logger, verbose_router_logger from litellm.caching.caching import DualCache, RedisCache -from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, is_redis_timeout_failure from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.constants import ( _REALTIME_BODY_CACHE_SIZE, @@ -3411,7 +3411,7 @@ async def _apply_spend_counter_increments(pending: Sequence[_PendingSpendIncreme results: Final = await redis_cache.async_increment_pipeline(increment_list=increment_list) except Exception as e: await asyncio.gather(*(_invalidate_spend_counter(counter_key=item.counter_key) for item in pending)) - if isinstance(e, RedisCircuitBreakerOpenError): + if isinstance(e, RedisCircuitBreakerOpenError) or is_redis_timeout_failure(e): return raise for item, current_value in zip(pending, results or ()): diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index d0974b2420c..5840f450ac6 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -977,17 +977,17 @@ async def test_stale_timeout_does_not_let_sub_threshold_hard_failures_open_the_b from redis.exceptions import ConnectionError as RedisConnectionError from redis.exceptions import TimeoutError as RedisTimeoutError - from litellm.caching.redis_cache import RedisCircuitBreaker, _is_redis_timeout_failure + from litellm.caching.redis_cache import RedisCircuitBreaker, is_redis_timeout_failure breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60, timeout_min_duration=0.05) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisTimeoutError("read timed out"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisTimeoutError("read timed out"))) await asyncio.sleep(0.06) for _ in range(breaker.failure_threshold - 1): - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisConnectionError("refused"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisConnectionError("refused"))) assert breaker.is_open() is False, "2 hard failures and 1 stale timeout are below both thresholds" - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisConnectionError("refused"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisConnectionError("refused"))) assert breaker.is_open() is True, "the threshold-th hard failure must still open it" @@ -999,19 +999,19 @@ async def test_hard_failure_resets_timeout_streak_so_a_later_burst_must_earn_its from redis.exceptions import ConnectionError as RedisConnectionError from redis.exceptions import TimeoutError as RedisTimeoutError - from litellm.caching.redis_cache import RedisCircuitBreaker, _is_redis_timeout_failure + from litellm.caching.redis_cache import RedisCircuitBreaker, is_redis_timeout_failure breaker = RedisCircuitBreaker(failure_threshold=3, recovery_timeout=60, timeout_min_duration=0.05) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisTimeoutError("read timed out"))) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisConnectionError("refused"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisTimeoutError("read timed out"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisConnectionError("refused"))) await asyncio.sleep(0.06) for _ in range(breaker.failure_threshold): - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisTimeoutError("read timed out"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisTimeoutError("read timed out"))) assert breaker.is_open() is False, "the burst is instantaneous, so the duration gate must hold it closed" await asyncio.sleep(0.06) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisTimeoutError("read timed out"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisTimeoutError("read timed out"))) assert breaker.is_open() is True, "the same run of timeouts persisting past the duration must open it" @@ -1022,7 +1022,7 @@ async def test_breaker_metrics_track_state_and_failure_class(): from redis.exceptions import ConnectionError as RedisConnectionError from redis.exceptions import TimeoutError as RedisTimeoutError - from litellm.caching.redis_cache import RedisCircuitBreaker, _is_redis_timeout_failure + from litellm.caching.redis_cache import RedisCircuitBreaker, is_redis_timeout_failure def sample(name, labels=None): return REGISTRY.get_sample_value(name, labels) or 0.0 @@ -1034,9 +1034,9 @@ async def test_breaker_metrics_track_state_and_failure_class(): closed_gauge_before = sample("litellm_redis_circuit_breaker_state", {"state": "closed"}) breaker = RedisCircuitBreaker(failure_threshold=2, recovery_timeout=60, timeout_min_duration=5.0) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisTimeoutError("t"))) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisConnectionError("refused"))) - breaker.record_failure(is_timeout=_is_redis_timeout_failure(RedisConnectionError("refused"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisTimeoutError("t"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisConnectionError("refused"))) + breaker.record_failure(is_timeout=is_redis_timeout_failure(RedisConnectionError("refused"))) assert sample("litellm_redis_circuit_breaker_failures_total", {"failure_class": "timeout"}) == timeout_before + 1 assert sample("litellm_redis_circuit_breaker_failures_total", {"failure_class": "connectivity"}) == hard_before + 2 diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index 2f47736a398..19b5a11af33 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -1180,6 +1180,28 @@ async def test_apply_spend_counter_increments_open_breaker_invalidates_and_retur fake_cache.in_memory_cache.set_cache.assert_not_called() +@pytest.mark.asyncio +async def test_apply_spend_counter_increments_redis_timeout_invalidates_and_returns(monkeypatch): + """A Redis timeout is the streak the breaker is already counting and the cache layer already logged. + + Re-raising it sent every request in the pre-open window through the cost callback's error + path, which logged a traceback and fired the failed-tracking alert once per request. + """ + from redis.exceptions import TimeoutError as RedisTimeoutError + + fake_cache = _make_spend_counter_cache() + fake_cache.redis_cache.async_increment_pipeline = AsyncMock( + side_effect=RedisTimeoutError("Timeout reading from 127.0.0.1:6379") + ) + monkeypatch.setattr(ps, "spend_counter_cache", fake_cache) + + await ps._apply_spend_counter_increments(_two_pending_increments()) + + deleted_keys = sorted(call.kwargs["key"] for call in fake_cache.in_memory_cache.delete_cache.call_args_list) + assert deleted_keys == ["spend:key:k", "spend:team:t"] + fake_cache.in_memory_cache.set_cache.assert_not_called() + + @pytest.mark.asyncio async def test_apply_spend_counter_increments_other_redis_error_invalidates_and_raises(monkeypatch): fake_cache = _make_spend_counter_cache() From f681a978f06baa13da0f0c24f7b1ac3a20d9a02a Mon Sep 17 00:00:00 2001 From: yassin Date: Sat, 12 Sep 2026 01:37:36 +0000 Subject: [PATCH 010/114] fix(redis): use a monotonic clock for the timeout log throttle and trim test docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_cache.py | 2 +- tests/test_litellm/caching/test_dual_cache.py | 7 +------ tests/test_litellm/caching/test_redis_cache.py | 9 +-------- .../proxy/proxy_server/test_spend_counters.py | 6 +----- 4 files changed, 4 insertions(+), 20 deletions(-) diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 7a3e689a667..e5e27d1e02b 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -409,7 +409,7 @@ class RedisCircuitBreakerOpenError(Exception): class _RedisTimeoutLogThrottle: """Admits one Redis timeout log line per interval and counts the timeouts it suppressed in between.""" - def __init__(self, interval: float, clock: Callable[[], float] = time.time) -> None: + def __init__(self, interval: float, clock: Callable[[], float] = time.monotonic) -> None: self.interval = interval self._clock = clock self._lock = threading.Lock() diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 850fa14106b..6f29be00b30 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -708,12 +708,7 @@ async def test_open_breaker_keeps_async_batch_read_memory_hits_and_releases_rese @pytest.mark.asyncio async def test_redis_timeouts_falling_back_to_memory_log_once_per_interval(caplog, monkeypatch): - """The in-memory fallback WARNING must not repeat for every timed-out increment during a blip. - - The rate limiter's pipeline increments and the dual cache increments each logged a WARNING per - call while Redis timed out, hundreds of lines per second before the breaker opened. The first - timeout of a streak keeps its WARNING, the rest are DEBUG until the summary interval passes. - """ + """The first fallback WARNING of a timeout streak logs, the rest stay at DEBUG until the summary.""" from redis.exceptions import TimeoutError as RedisTimeoutError from litellm.caching import redis_cache as redis_cache_module diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 5840f450ac6..4ca33894aed 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1205,14 +1205,7 @@ async def test_a_probe_overtaken_by_a_later_outage_leaves_the_breaker_to_the_new def test_timeouts_during_a_blip_log_once_per_interval_not_once_per_call(sync_batch_redis_cache, caplog, monkeypatch): - """A Redis latency blip must not write one ERROR line per timed-out cache call. - - Before the breaker opens (up to REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION of timeouts) every - cache operation logged its own ERROR or WARNING line, so one single-worker proxy wrote - ~1100 lines in 5 s at LITELLM_LOG=WARNING. A timeout streak now logs its first failure, then - one summary line per REDIS_TIMEOUT_LOG_INTERVAL carrying the count of suppressed timeouts, - while every timeout stays visible at DEBUG. Hard connectivity failures keep their per-call line. - """ + """A timeout streak logs its first failure plus one summary per interval; other failures log per call.""" import logging from redis.exceptions import TimeoutError as RedisTimeoutError diff --git a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py index 19b5a11af33..4a1fc389d3e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_spend_counters.py +++ b/tests/test_litellm/proxy/proxy_server/test_spend_counters.py @@ -1182,11 +1182,7 @@ async def test_apply_spend_counter_increments_open_breaker_invalidates_and_retur @pytest.mark.asyncio async def test_apply_spend_counter_increments_redis_timeout_invalidates_and_returns(monkeypatch): - """A Redis timeout is the streak the breaker is already counting and the cache layer already logged. - - Re-raising it sent every request in the pre-open window through the cost callback's error - path, which logged a traceback and fired the failed-tracking alert once per request. - """ + """A Redis timeout invalidates the counters and returns without reaching the cost callback's error path.""" from redis.exceptions import TimeoutError as RedisTimeoutError fake_cache = _make_spend_counter_cache() From 28f2d1f0168aa31639a23447d391516129267069 Mon Sep 17 00:00:00 2001 From: yassin Date: Sat, 12 Sep 2026 01:58:36 +0000 Subject: [PATCH 011/114] test(redis): cover the write and list timeout paths going through the shared log throttle Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_litellm/caching/test_redis_cache.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py index 4ca33894aed..5ea21dae539 100644 --- a/tests/test_litellm/caching/test_redis_cache.py +++ b/tests/test_litellm/caching/test_redis_cache.py @@ -1252,3 +1252,60 @@ def test_timeouts_during_a_blip_log_once_per_interval_not_once_per_call(sync_bat for _ in range(3): assert sync_batch_redis_cache.get_cache("lit7520") is None assert [r.levelno for r in caplog.records if "redis unavailable" in r.getMessage()] == [logging.ERROR] * 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_method", + [ + pytest.param(lambda c: c.async_set_cache_pipeline([("lit7520", "v")]), id="async_set_cache_pipeline"), + pytest.param(lambda c: c.async_set_cache_sadd("lit7520", ["v"], ttl=None), id="async_set_cache_sadd"), + pytest.param(lambda c: c.async_increment("lit7520", 1.0), id="async_increment"), + pytest.param( + lambda c: c.async_increment_pipeline([{"key": "lit7520", "increment_value": 1.0, "ttl": 60}]), + id="async_increment_pipeline", + ), + pytest.param(lambda c: c.async_rpush("lit7520", ["v"]), id="async_rpush"), + pytest.param( + lambda c: c.async_rpush_pipeline([{"key": "lit7520", "values": ["v"]}]), id="async_rpush_pipeline" + ), + pytest.param(lambda c: c.async_lpop("lit7520"), id="async_lpop"), + pytest.param(lambda c: c.async_lpop_pipeline([{"key": "lit7520", "count": 1}]), id="async_lpop_pipeline"), + ], +) +async def test_write_path_timeouts_inside_the_interval_stay_at_debug(call_method, caplog, monkeypatch, redis_no_ping): + """A write or list operation timing out mid-streak is counted by the throttle instead of logging its own ERROR.""" + import contextlib + import logging + + from redis.exceptions import TimeoutError as RedisTimeoutError + + from litellm.caching import redis_cache as redis_cache_module + from litellm.caching.redis_cache import _RedisTimeoutLogThrottle + + clock = MagicMock(return_value=1_000.0) + throttle = _RedisTimeoutLogThrottle(interval=5.0, clock=clock) + assert throttle.admit() == 0 + monkeypatch.setattr(redis_cache_module, "_redis_timeout_log_throttle", throttle) + + timeout = RedisTimeoutError("Timeout reading from 127.0.0.1:6379") + client = MagicMock() + client.pipeline.return_value.__aenter__.side_effect = timeout + client.sadd = AsyncMock(side_effect=timeout) + client.incrbyfloat = AsyncMock(side_effect=timeout) + client.rpush = AsyncMock(side_effect=timeout) + client.lpop = AsyncMock(side_effect=timeout) + monkeypatch.setenv("REDIS_HOST", "https://my-test-host") + cache = RedisCache() + + with ( + patch.object(cache, "init_async_client", return_value=client), + caplog.at_level(logging.DEBUG, logger="LiteLLM"), + ): + with contextlib.suppress(RedisTimeoutError): + await call_method(cache) + + timeout_records = [r for r in caplog.records if "Timeout reading from" in r.getMessage()] + assert [(r.levelno, r.filename) for r in timeout_records] == [(logging.DEBUG, "redis_cache.py")] + clock.return_value += 5.0 + assert throttle.admit() == 1 From 3c2342bfd32edddadfa3429ddefa159fdd9e32c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 12 Sep 2026 15:05:41 -0700 Subject: [PATCH 012/114] refactor(cost): return a new prompt token details wrapper when combining usage --- litellm/cost_calculator.py | 55 +++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b865318f3af..50118af8a30 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2326,33 +2326,29 @@ def _combine_cached_tokens_details( ) -def _combine_prompt_tokens_details(combined: Usage, usage: Usage) -> None: - if not (hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details): - return - if not hasattr(combined, "prompt_tokens_details") or not combined.prompt_tokens_details: - combined.prompt_tokens_details = PromptTokensDetailsWrapper() - - for attr in _summable_prompt_token_fields(usage.prompt_tokens_details): - if ( - hasattr(usage.prompt_tokens_details, attr) - and not attr.startswith("_") - and not callable(_attribute_value(usage.prompt_tokens_details, attr)) - ): - current_val = getattr(combined.prompt_tokens_details, attr, 0) or 0 - new_val = getattr(usage.prompt_tokens_details, attr, 0) or 0 - if new_val is not None and isinstance(new_val, (int, float)): - setattr( - combined.prompt_tokens_details, - attr, - current_val + new_val, - ) - - new_cached_tokens_details: Final = getattr(usage.prompt_tokens_details, "cached_tokens_details", None) - if isinstance(new_cached_tokens_details, CachedTokensDetails): - combined.prompt_tokens_details.cached_tokens_details = _combine_cached_tokens_details( - getattr(combined.prompt_tokens_details, "cached_tokens_details", None), - new_cached_tokens_details, - ) +def _combine_prompt_tokens_details( + current: PromptTokensDetailsWrapper | None, new: PromptTokensDetailsWrapper +) -> PromptTokensDetailsWrapper: + base: Final = current if current is not None else PromptTokensDetailsWrapper() + base_values: Final = MappingProxyType( + {attr: getattr(base, attr) for attr in type(base).model_fields if hasattr(base, attr)} + ) + summed: Final = MappingProxyType( + { + attr: (getattr(base, attr, 0) or 0) + (getattr(new, attr) or 0) + for attr in _summable_prompt_token_fields(new) + if hasattr(new, attr) and isinstance(getattr(new, attr) or 0, (int, float)) + } + ) + new_cached_tokens_details: Final = getattr(new, "cached_tokens_details", None) + cached_tokens_details: Final = ( + _combine_cached_tokens_details(getattr(base, "cached_tokens_details", None), new_cached_tokens_details) + if isinstance(new_cached_tokens_details, CachedTokensDetails) + else getattr(base, "cached_tokens_details", None) + ) + return PromptTokensDetailsWrapper( + **MappingProxyType({**base_values, **summed, "cached_tokens_details": cached_tokens_details}) + ) class BaseTokenUsageProcessor: @@ -2381,7 +2377,10 @@ class BaseTokenUsageProcessor: and isinstance(current_val, (int, float)) ): setattr(combined, attr, current_val + new_val) - _combine_prompt_tokens_details(combined, usage) + if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: + combined.prompt_tokens_details = _combine_prompt_tokens_details( + getattr(combined, "prompt_tokens_details", None), usage.prompt_tokens_details + ) # Handle nested completion_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: From e4b05883627555fd72db16f0ecf376b75a6d93a3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 12 Sep 2026 15:31:53 -0700 Subject: [PATCH 013/114] fix(cost): carry cache_read_input_audio_token_cost through get_model_info Every proxy and router cost lookup goes through get_model_info, which copies cost map keys explicitly, so the new audio cache-read branch always fell back to the text cache-read rate there. Copy the key so models whose audio cache-read rate differs from the text one bill cached audio correctly. --- litellm/types/utils.py | 1 + litellm/utils.py | 1 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 17 +++++++++++++++++ tests/test_litellm/test_utils.py | 8 ++++++++ 4 files changed, 27 insertions(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9eb70e189a1..ef191d79177 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -251,6 +251,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None + cache_read_input_audio_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing diff --git a/litellm/utils.py b/litellm/utils.py index 1a77655a5a4..d048265ecd3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5866,6 +5866,7 @@ def _get_model_info_helper( "cache_creation_input_token_cost_ultrafast", None ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), + cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 5d55dfb14a3..5ff1ab62698 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -5235,3 +5235,20 @@ def test_cached_audio_tokens_capped_at_cached_tokens(_local_model_cost_map: None model="gpt-realtime-2", usage=usage, custom_llm_provider="openai" ) assert prompt_cost == pytest.approx(116 * 4e-6 + (167 - 100) * 32e-6 + 100 * 4e-7) + + +def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_lookup(_local_model_cost_map: None) -> None: + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=600, + cached_tokens=500, + cached_tokens_details={"text_tokens": 100, "audio_tokens": 400}, + ), + ) + + prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai") + assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 19ed31c7b22..d9ca7d9ddee 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -6430,3 +6430,11 @@ def test_completion_finishes_response_metadata_before_handing_the_response_to_th assert snapshot["litellm_call_id"] assert snapshot["response_cost"] is not None assert snapshot["api_base"] + + +def test_get_model_info_carries_cache_read_input_audio_token_cost(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + info = litellm.get_model_info("gpt-realtime-2.1-mini", custom_llm_provider="openai") + assert info["cache_read_input_audio_token_cost"] == 3e-07 + assert info["cache_read_input_token_cost"] == 6e-08 From 305caa8260fb98f821aa69eeba466a162b69c1c4 Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 12 Sep 2026 23:01:41 +0000 Subject: [PATCH 014/114] test: drop unrelated reformatting from merge resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_cost_calculator.py | 335 +++++++++++++++------ 1 file changed, 241 insertions(+), 94 deletions(-) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index f2ee2cdbd9a..c23f5c08a70 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,3 +1,4 @@ + import json from pathlib import Path from typing import Final @@ -148,7 +149,9 @@ def test_jina_rerank_bills_total_tokens_at_input_rate_only(_local_model_cost_map def test_cost_calculator_with_response_cost_in_additional_headers(): class MockResponse(BaseModel): - _hidden_params = {"additional_headers": {"llm_provider-x-litellm-response-cost": 1000}} + _hidden_params = { + "additional_headers": {"llm_provider-x-litellm-response-cost": 1000} + } result = response_cost_calculator( response_object=MockResponse(), @@ -204,9 +207,7 @@ def test_vertex_lyria_speech_cost( call_type=call_type, ) - expected: Final = ( - 0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1) - ) + expected: Final = 0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1) assert cost == pytest.approx(expected) @@ -333,12 +334,13 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): # Step 1: Test a model where input_cost_per_image_token is not set. # In this case the calculation should use input_cost_per_token as fallback. - assert model_info.get("input_cost_per_image_token") is None, ( - "Test case expects that input_cost_per_image_token is not set" - ) + assert ( + model_info.get("input_cost_per_image_token") is None + ), "Test case expects that input_cost_per_image_token is not set" expected_cost = ( - usage.prompt_tokens_details.audio_tokens * model_info["input_cost_per_audio_token"] + usage.prompt_tokens_details.audio_tokens + * model_info["input_cost_per_audio_token"] + usage.prompt_tokens_details.text_tokens * model_info["input_cost_per_token"] + usage.prompt_tokens_details.image_tokens * model_info["input_cost_per_token"] + usage.completion_tokens * model_info["output_cost_per_token"] @@ -373,9 +375,12 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): ) expected_cost = ( - usage.prompt_tokens_details.audio_tokens * temp_model_info_object["input_cost_per_audio_token"] - + usage.prompt_tokens_details.text_tokens * temp_model_info_object["input_cost_per_token"] - + usage.prompt_tokens_details.image_tokens * temp_model_info_object["input_cost_per_image_token"] + usage.prompt_tokens_details.audio_tokens + * temp_model_info_object["input_cost_per_audio_token"] + + usage.prompt_tokens_details.text_tokens + * temp_model_info_object["input_cost_per_token"] + + usage.prompt_tokens_details.image_tokens + * temp_model_info_object["input_cost_per_image_token"] + usage.completion_tokens * temp_model_info_object["output_cost_per_token"] ) @@ -385,11 +390,14 @@ def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): def test_transcription_cost_uses_token_pricing(_local_model_cost_map): from litellm import completion_cost + usage = Usage( prompt_tokens=14, completion_tokens=45, total_tokens=59, - prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=0, audio_tokens=14), + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, audio_tokens=14 + ), ) response = TranscriptionResponse(text="demo text") response.usage = usage @@ -433,6 +441,7 @@ def test_transcription_token_pricing_is_provider_aware(_local_model_cost_map): def test_transcription_cost_falls_back_to_duration(_local_model_cost_map): from litellm import completion_cost + response = TranscriptionResponse(text="demo text") response.duration = 10.0 @@ -453,6 +462,7 @@ def test_vertex_chirp_3_transcription_cost_from_duration(_local_model_cost_map): every transcription priced to $0.00 instead of using input_cost_per_second.""" from litellm import completion_cost + response = TranscriptionResponse(text="demo text") response.duration = 18.0 @@ -476,7 +486,9 @@ def test_handle_realtime_stream_cost_calculation(): {"type": "session.created", "session": {"model": "gpt-3.5-turbo"}}, { "type": "response.done", - "response": {"usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}}, + "response": { + "usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150} + }, }, { "type": "response.done", @@ -507,7 +519,9 @@ def test_handle_realtime_stream_cost_calculation(): expected_cost = (300 * 0.0015 / 1000) + ( # input tokens (100 + 200) 150 * 0.002 / 1000 ) # output tokens (50 + 100) - assert abs(cost - expected_cost) <= 0.00075 # Allow small floating point differences + assert ( + abs(cost - expected_cost) <= 0.00075 + ) # Allow small floating point differences # Test with different model name in session results[0]["session"]["model"] = "gpt-4" @@ -587,7 +601,14 @@ def test_handle_realtime_stream_cost_calculation_stores_cost_breakdown(): assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["input_cost"] > 0 assert logging_obj.cost_breakdown["output_cost"] > 0 - assert abs(logging_obj.cost_breakdown["input_cost"] + logging_obj.cost_breakdown["output_cost"] - total_cost) < 1e-9 + assert ( + abs( + logging_obj.cost_breakdown["input_cost"] + + logging_obj.cost_breakdown["output_cost"] + - total_cost + ) + < 1e-9 + ) assert abs(logging_obj.cost_breakdown["total_cost"] - total_cost) < 1e-9 @@ -661,7 +682,9 @@ def test_realtime_logging_object_allows_null_transcript_in_conversation_item_add }, ] - usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results) + usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results + ) logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object( usage=usage, results=results, @@ -711,7 +734,9 @@ def test_realtime_logging_object_does_not_validate_unknown_event_types(): }, ] - usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results) + usage = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results + ) # On unfixed code this raises pydantic ValidationError instead of returning. logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object( usage=usage, @@ -723,7 +748,8 @@ def test_realtime_logging_object_does_not_validate_unknown_event_types(): unknown_types = { r["type"] for r in logging_result.results - if r["type"] in ("rate_limits.updated", "response.function_call_arguments.delta") + if r["type"] + in ("rate_limits.updated", "response.function_call_arguments.delta") } assert unknown_types == { "rate_limits.updated", @@ -756,7 +782,9 @@ def test_realtime_transcription_duration_cost(monkeypatch): "type": "session.created", "session": { "type": "transcription", - "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, + "audio": { + "input": {"transcription": {"model": "gpt-realtime-whisper"}} + }, }, }, { @@ -771,7 +799,9 @@ def test_realtime_transcription_duration_cost(monkeypatch): }, ] - combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results) + combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results + ) logging_obj = Logging( model="gpt-realtime-whisper", messages=[], @@ -864,7 +894,9 @@ def test_realtime_transcription_token_billed_fallback(monkeypatch): # gpt-4o-transcribe: input_cost_per_audio_token = 2.5e-06, input_cost_per_token = 2.5e-06, # output_cost_per_token = 1e-05 - model_info = litellm.get_model_info(model="gpt-4o-transcribe", custom_llm_provider="openai") + model_info = litellm.get_model_info( + model="gpt-4o-transcribe", custom_llm_provider="openai" + ) usage = { "type": "tokens", "input_tokens": 40, @@ -945,7 +977,10 @@ def test_get_transcription_model_falls_back_to_session_model(monkeypatch): mock_response=True, ) - assert result._hidden_params["response_cost"] > result_2._hidden_params["response_cost"] + assert ( + result._hidden_params["response_cost"] + > result_2._hidden_params["response_cost"] + ) model_info = router.get_deployment_model_info( model_id="my-unique-model-id", model_name="anthropic/claude-sonnet-4-5-20250929" @@ -1108,7 +1143,9 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): assert entry.get("input_cost_per_token") is None assert entry.get("tiered_pricing") is not None # The stripped shared alias must not carry tiered pricing. - assert litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None + assert ( + litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None + ) selected = _select_model_name_for_cost_calc( model="dashscope/qwen-tier-only-test", @@ -1188,7 +1225,9 @@ def test_azure_realtime_cost_calculator(_local_model_cost_map): combined_usage_object=Usage( prompt_tokens=100, completion_tokens=100, - prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=10, audio_tokens=90), + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=10, audio_tokens=90 + ), ), custom_llm_provider="azure", litellm_model_name="my-custom-azure-deployment", @@ -1207,6 +1246,7 @@ def test_azure_audio_output_cost_calculation(_local_model_cost_map): """ from litellm.types.utils import Choices, CompletionTokensDetailsWrapper, Message + # Scenario from issue #19764: # Input: 17 text tokens, 0 audio tokens # Output: 110 text tokens, 482 audio tokens @@ -1262,10 +1302,14 @@ def test_azure_audio_output_cost_calculation(_local_model_cost_map): wrong_total_cost = expected_input_cost + wrong_output_cost # Verify audio tokens are NOT charged at text rate (the bug) - assert abs(cost - wrong_total_cost) > 0.001, "Bug: Audio tokens are being charged at text token rate" + assert ( + abs(cost - wrong_total_cost) > 0.001 + ), "Bug: Audio tokens are being charged at text token rate" # Verify cost matches - assert abs(cost - expected_total_cost) < 0.0000001, f"Expected cost {expected_total_cost}, got {cost}" + assert ( + abs(cost - expected_total_cost) < 0.0000001 + ), f"Expected cost {expected_total_cost}, got {cost}" def test_default_image_cost_calculator(monkeypatch): @@ -1279,7 +1323,9 @@ def test_default_image_cost_calculator(monkeypatch): monkeypatch.setattr( litellm, "model_cost", - {"azure/bf9001cd7209f5734ecb4ab937a5a0e2ba5f119708bd68f184db362930f9dc7b": temp_object}, + { + "azure/bf9001cd7209f5734ecb4ab937a5a0e2ba5f119708bd68f184db362930f9dc7b": temp_object + }, ) args = { @@ -1495,7 +1541,9 @@ def test_gemini_25_implicit_caching_cost(): expected_cost = 0.00068708 # Allow for small floating point differences - assert abs(result - expected_cost) < 1e-8, f"Expected cost {expected_cost}, but got {result}" + assert ( + abs(result - expected_cost) < 1e-8 + ), f"Expected cost {expected_cost}, but got {result}" print(f"✓ Gemini 2.5 implicit caching cost calculation is correct: ${result:.8f}") @@ -1566,7 +1614,9 @@ def test_log_context_cost_calculation(): # Get model info to understand the pricing from litellm import get_model_info - model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic") + model_info = get_model_info( + model="claude-4-sonnet-20250514", custom_llm_provider="anthropic" + ) # Calculate expected cost based on actual model pricing input_cost_per_token = model_info.get("input_cost_per_token", 0) @@ -1574,8 +1624,12 @@ def test_log_context_cost_calculation(): cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0) # Check if tiered pricing is applied - input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token) - output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token) + input_cost_above_200k = model_info.get( + "input_cost_per_token_above_200k_tokens", input_cost_per_token + ) + output_cost_above_200k = model_info.get( + "output_cost_per_token_above_200k_tokens", output_cost_per_token + ) cache_creation_above_200k = model_info.get( "cache_creation_input_token_cost_above_200k_tokens", cache_creation_cost_per_token, @@ -1583,23 +1637,31 @@ def test_log_context_cost_calculation(): print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}") print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}") - print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}") + print( + f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}" + ) # Handle tiered pricing - if not available, use base pricing if input_cost_above_200k is not None: - print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}") + print( + f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}" + ) else: print("DEBUG: No tiered input pricing available, using base pricing") input_cost_above_200k = input_cost_per_token if output_cost_above_200k is not None: - print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}") + print( + f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}" + ) else: print("DEBUG: No tiered output pricing available, using base pricing") output_cost_above_200k = output_cost_per_token if cache_creation_above_200k is not None: - print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}") + print( + f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}" + ) else: print("DEBUG: No tiered cache creation pricing available, using base pricing") cache_creation_above_200k = cache_creation_cost_per_token @@ -1613,9 +1675,13 @@ def test_log_context_cost_calculation(): print(f"DEBUG: Expected total: ${expected_total:.6f}") # Allow for small floating point differences - assert abs(result - expected_total) < 1e-6, f"Expected cost ${expected_total:.6f}, but got ${result:.6f}" + assert ( + abs(result - expected_total) < 1e-6 + ), f"Expected cost ${expected_total:.6f}, but got ${result:.6f}" - print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}") + print( + f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}" + ) print(f" - Input tokens (300k): ${expected_input_cost:.6f}") print(f" - Output tokens (50k): ${expected_output_cost:.6f}") print(f" - Cache creation (1k): ${expected_cache_cost:.6f}") @@ -1674,7 +1740,8 @@ def test_gemini_25_explicit_caching_cost_direct_usage(): expected_actual_cost = ( model_info["input_cost_per_token"] * usage.prompt_tokens_details.text_tokens - + model_info["cache_read_input_token_cost"] * usage.prompt_tokens_details.cached_tokens + + model_info["cache_read_input_token_cost"] + * usage.prompt_tokens_details.cached_tokens + model_info["output_cost_per_token"] * usage.completion_tokens ) @@ -1698,6 +1765,7 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map): from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.types.utils import PromptTokensDetailsWrapper, Usage + # Register a custom azure_ai model with cache pricing test_model_id = "test-azure-ai-claude-model" litellm.register_model( @@ -1746,12 +1814,13 @@ def test_azure_ai_cache_cost_calculation(_local_model_cost_map): print(f"Output cost: {output_cost}, Expected: {expected_output_cost}") print(f"Total cost: {total_cost}") - assert abs(input_cost - expected_input_cost) < 1e-10, ( - f"Input cost mismatch: got {input_cost}, expected {expected_input_cost}" - ) - assert abs(output_cost - expected_output_cost) < 1e-10, ( - f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}" - ) + assert ( + abs(input_cost - expected_input_cost) < 1e-10 + ), f"Input cost mismatch: got {input_cost}, expected {expected_input_cost}" + assert ( + abs(output_cost - expected_output_cost) < 1e-10 + ), f"Output cost mismatch: got {output_cost}, expected {expected_output_cost}" + AZURE_GPT_5_6_MAP_KEYS = ( @@ -1820,7 +1889,6 @@ def test_azure_gpt_5_6_rates_match_azure_price_page(_local_model_cost_map, model for key in token_cost_keys: assert entry[key] == pytest.approx(global_entry[key] * 1.1), key - def test_vertex_regional_deployment_costs_uplift_over_global(monkeypatch): """ Regression for https://github.com/BerriAI/litellm/issues/34393: two Vertex @@ -1903,6 +1971,7 @@ def test_cost_discount_vertex_ai(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response (use a model that exists in model_prices_and_context_window.json) response = ModelResponse( id="test-id", @@ -1931,6 +2000,7 @@ def test_cost_discount_vertex_ai(monkeypatch): custom_llm_provider="vertex_ai", ) + # Verify discount is applied (5% off means 95% of original cost) expected_cost = cost_without_discount * 0.95 assert cost_with_discount == pytest.approx(expected_cost, rel=1e-9) @@ -1948,6 +2018,7 @@ def test_cost_discount_not_applied_to_other_providers(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response for OpenAI response = ModelResponse( id="test-id", @@ -1976,6 +2047,7 @@ def test_cost_discount_not_applied_to_other_providers(monkeypatch): custom_llm_provider="openai", ) + # Costs should be the same (no discount applied to OpenAI) assert cost_with_selective_discount == cost_without_discount @@ -1991,6 +2063,7 @@ def test_cost_margin_percentage(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response response = ModelResponse( id="test-id", @@ -2019,6 +2092,7 @@ def test_cost_margin_percentage(monkeypatch): custom_llm_provider="openai", ) + # Verify margin is applied (10% margin means 110% of original cost) expected_cost = cost_without_margin * 1.10 assert cost_with_margin == pytest.approx(expected_cost, rel=1e-9) @@ -2036,6 +2110,7 @@ def test_cost_margin_fixed_amount(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response response = ModelResponse( id="test-id", @@ -2064,6 +2139,7 @@ def test_cost_margin_fixed_amount(monkeypatch): custom_llm_provider="openai", ) + # Verify fixed margin is applied expected_cost = cost_without_margin + 0.001 assert cost_with_margin == pytest.approx(expected_cost, rel=1e-9) @@ -2081,6 +2157,7 @@ def test_cost_margin_combined(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response response = ModelResponse( id="test-id", @@ -2100,7 +2177,9 @@ def test_cost_margin_combined(monkeypatch): ) # Set 8% margin + $0.0005 fixed for openai - monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"percentage": 0.08, "fixed_amount": 0.0005}}) + monkeypatch.setattr(litellm, "cost_margin_config", { + "openai": {"percentage": 0.08, "fixed_amount": 0.0005} + }) # Calculate cost with margin cost_with_margin = completion_cost( @@ -2109,6 +2188,7 @@ def test_cost_margin_combined(monkeypatch): custom_llm_provider="openai", ) + # Verify combined margin is applied expected_cost = cost_without_margin * 1.08 + 0.0005 assert cost_with_margin == pytest.approx(expected_cost, rel=1e-9) @@ -2126,6 +2206,7 @@ def test_cost_margin_global(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response response = ModelResponse( id="test-id", @@ -2154,6 +2235,7 @@ def test_cost_margin_global(monkeypatch): custom_llm_provider="openai", ) + # Verify global margin is applied expected_cost = cost_without_margin * 1.05 assert cost_with_global_margin == pytest.approx(expected_cost, rel=1e-9) @@ -2171,6 +2253,7 @@ def test_cost_margin_provider_overrides_global(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response response = ModelResponse( id="test-id", @@ -2199,13 +2282,16 @@ def test_cost_margin_provider_overrides_global(monkeypatch): custom_llm_provider="openai", ) + # Verify provider-specific margin is used (not global) expected_cost = cost_without_margin * 1.10 # 10% from provider, not 5% from global assert cost_with_provider_margin == pytest.approx(expected_cost, rel=1e-9) print("✓ Cost margin provider override test passed:") print(f" - Original cost: ${cost_without_margin:.6f}") - print(f" - Cost with provider margin (10%, overrides 5% global): ${cost_with_provider_margin:.6f}") + print( + f" - Cost with provider margin (10%, overrides 5% global): ${cost_with_provider_margin:.6f}" + ) print(f" - Margin added: ${cost_with_provider_margin - cost_without_margin:.6f}") @@ -2216,6 +2302,7 @@ def test_cost_margin_with_discount(monkeypatch): from litellm import completion_cost from litellm.types.utils import Usage + # Create mock response response = ModelResponse( id="test-id", @@ -2246,6 +2333,7 @@ def test_cost_margin_with_discount(monkeypatch): custom_llm_provider="openai", ) + # Verify: discount applied first, then margin # Base cost -> discount: base * 0.95 -> margin: (base * 0.95) * 1.10 expected_cost = base_cost * 0.95 * 1.10 @@ -2283,7 +2371,9 @@ def test_azure_image_generation_cost_calculator(): size=None, usage=ImageUsage( input_tokens=0, - input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), + input_tokens_details=ImageUsageInputTokensDetails( + image_tokens=0, text_tokens=0 + ), output_tokens=0, total_tokens=0, ), @@ -2313,6 +2403,7 @@ def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_m """Test that completion_cost extracts service_tier from completion_response object.""" from litellm import completion_cost + # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -2353,18 +2444,23 @@ def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_m assert flex_cost < standard_cost, "Flex cost should be less than standard cost" flex_ratio = flex_cost / standard_cost - assert 0.45 <= flex_ratio <= 0.55, f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" + assert ( + 0.45 <= flex_ratio <= 0.55 + ), f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map): """Test that completion_cost extracts service_tier from usage object.""" from litellm import completion_cost + # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" # Create usage object with service_tier - usage_with_service_tier = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + usage_with_service_tier = Usage( + prompt_tokens=1000, completion_tokens=500, total_tokens=1500 + ) # Set service_tier as an attribute on the usage object setattr(usage_with_service_tier, "service_tier", "flex") @@ -2382,7 +2478,9 @@ def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map) ) # Create usage object without service_tier - usage_without_service_tier = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + usage_without_service_tier = Usage( + prompt_tokens=1000, completion_tokens=500, total_tokens=1500 + ) # Create ModelResponse with usage without service_tier response_standard = ModelResponse( @@ -2403,13 +2501,16 @@ def test_completion_cost_extracts_service_tier_from_usage(_local_model_cost_map) assert flex_cost < standard_cost, "Flex cost should be less than standard cost" flex_ratio = flex_cost / standard_cost - assert 0.45 <= flex_ratio <= 0.55, f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" + assert ( + 0.45 <= flex_ratio <= 0.55 + ), f"Flex pricing should be ~50% of standard, got {flex_ratio:.2f}" def test_completion_cost_service_tier_priority(_local_model_cost_map): """Test that service_tier extraction follows priority: optional_params > completion_response > usage.""" from litellm import completion_cost + # Test with gpt-5-nano which has flex pricing model = "gpt-5-nano" @@ -2458,13 +2559,16 @@ def test_completion_cost_service_tier_priority(_local_model_cost_map): assert cost_from_usage > 0, "Cost from usage should be greater than 0" # Costs should be similar (all using flex) - assert abs(cost_from_params - cost_from_usage) < 1e-6, "Costs from params and usage should be similar (both flex)" + assert ( + abs(cost_from_params - cost_from_usage) < 1e-6 + ), "Costs from params and usage should be similar (both flex)" def test_completion_cost_service_tier_for_bedrock(_local_model_cost_map): """Test that Bedrock cost calculation applies service_tier-specific pricing.""" from litellm import completion_cost + model = "bedrock/us-east-1/test-bedrock-service-tier-cost-model" litellm.register_model( model_cost={ @@ -2520,6 +2624,7 @@ def test_completion_cost_service_tier_for_anthropic(_local_model_cost_map): from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig + model = "claude-test-service-tier-cost-model" litellm.register_model( model_cost={ @@ -2572,6 +2677,7 @@ def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(_local_mo from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig + model = "claude-test-auto-tier-cost-model" litellm.register_model( model_cost={ @@ -2665,6 +2771,7 @@ def test_completion_cost_non_string_service_tier_defers_to_served_tier(_local_mo from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig + model = "claude-test-non-string-tier-cost-model" litellm.register_model( model_cost={ @@ -2714,6 +2821,7 @@ def test_completion_cost_non_string_response_service_tier_defers_to_served_tier( from litellm import completion_cost from litellm.llms.anthropic.chat.transformation import AnthropicConfig + model = "claude-test-response-non-string-tier-cost-model" litellm.register_model( model_cost={ @@ -2736,7 +2844,9 @@ def test_completion_cost_non_string_response_service_tier_defers_to_served_tier( }, reasoning_content=None, ) - response = ModelResponse(usage=usage, model=model, service_tier={"name": "priority"}) + response = ModelResponse( + usage=usage, model=model, service_tier={"name": "priority"} + ) cost = completion_cost( completion_response=response, @@ -2759,6 +2869,7 @@ def test_completion_cost_non_string_usage_service_tier_prices_standard(_local_mo """ from litellm import completion_cost + model = "claude-test-usage-non-string-tier-cost-model" litellm.register_model( model_cost={ @@ -2805,6 +2916,7 @@ def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(_l ) from litellm.types.utils import PromptTokensDetailsWrapper, Usage + model = "claude-test-priority-cache-fast-model" litellm.register_model( model_cost={ @@ -2830,7 +2942,9 @@ def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(_l ) usage.speed = "fast" - prompt_cost, completion_cost = anthropic_cost_per_token(model=model, usage=usage, service_tier="priority") + prompt_cost, completion_cost = anthropic_cost_per_token( + model=model, usage=usage, service_tier="priority" + ) expected_prompt = ((1000 - 200) * 6e-6 + 200 * 0.6e-6) * 2 expected_completion = 500 * 30e-6 * 2 @@ -2960,7 +3074,9 @@ def test_anthropic_fast_multiplier_only_on_models_with_fast_mode(_local_model_co "model", ["claude-sonnet-4-6", "claude-mythos-5", "claude-mythos-preview"], ) -def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_local_model_cost_map, monkeypatch, model): +def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models( + _local_model_cost_map, monkeypatch, model +): """ Anthropic bills every Claude 4.6+ model served with ``inference_geo="us"`` at 1.1x, and echoes that geo back in the response usage, so each of these real @@ -3025,26 +3141,28 @@ def test_gemini_cache_tokens_details_no_negative_values(): usage = VertexGeminiConfig._calculate_usage(completion_response) # Text tokens should be non-cached text only: 9402 - 9393 = 9 - assert usage.prompt_tokens_details.text_tokens == 9, ( - f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}" - ) + assert ( + usage.prompt_tokens_details.text_tokens == 9 + ), f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}" # Image tokens should be non-cached image only: 258 - 258 = 0 - assert usage.prompt_tokens_details.image_tokens == 0, ( - f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}" - ) + assert ( + usage.prompt_tokens_details.image_tokens == 0 + ), f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}" # Total cached should match - assert usage.prompt_tokens_details.cached_tokens == 9651, ( - f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}" - ) + assert ( + usage.prompt_tokens_details.cached_tokens == 9651 + ), f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}" # MOST IMPORTANT: text_tokens should NEVER be negative - assert usage.prompt_tokens_details.text_tokens >= 0, ( - f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750" - ) + assert ( + usage.prompt_tokens_details.text_tokens >= 0 + ), f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750" - print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative") + print( + "✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative" + ) def test_gemini_without_cache_tokens_details(): @@ -3112,18 +3230,18 @@ def test_gemini_implicit_caching_cost_calculation(): usage = VertexGeminiConfig._calculate_usage(completion_response) # Verify parsing - assert usage.cache_read_input_tokens == 8000, ( - f"cache_read_input_tokens should be 8000, got {usage.cache_read_input_tokens}" - ) - assert usage.prompt_tokens_details.cached_tokens == 8000, ( - f"cached_tokens should be 8000, got {usage.prompt_tokens_details.cached_tokens}" - ) + assert ( + usage.cache_read_input_tokens == 8000 + ), f"cache_read_input_tokens should be 8000, got {usage.cache_read_input_tokens}" + assert ( + usage.prompt_tokens_details.cached_tokens == 8000 + ), f"cached_tokens should be 8000, got {usage.prompt_tokens_details.cached_tokens}" # CRITICAL: text_tokens should be (10000 - 8000) = 2000, NOT 10000 # This is the fix for issue #16341 - assert usage.prompt_tokens_details.text_tokens == 2000, ( - f"text_tokens should be 2000 (10000 - 8000), got {usage.prompt_tokens_details.text_tokens}" - ) + assert ( + usage.prompt_tokens_details.text_tokens == 2000 + ), f"text_tokens should be 2000 (10000 - 8000), got {usage.prompt_tokens_details.text_tokens}" # Verify cost calculation uses cached token pricing response = ModelResponse( @@ -3161,7 +3279,9 @@ def test_gemini_implicit_caching_cost_calculation(): f"Cached tokens may not be using reduced pricing." ) - print("✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly") + print( + "✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly" + ) def test_additional_costs_only_for_azure_ai(_local_model_cost_map): @@ -3175,6 +3295,7 @@ def test_additional_costs_only_for_azure_ai(_local_model_cost_map): """ from litellm.cost_calculator import _get_additional_costs + # Non-azure_ai providers should return None result = _get_additional_costs( model="gpt-4o", @@ -3317,7 +3438,12 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_prompt_details(): }, ) - expected = (4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125 + 100 * 0.000015 + expected = ( + (4000 - 1000 - 500) * 0.0000025 + + 1000 * 0.00000025 + + 500 * 0.000003125 + + 100 * 0.000015 + ) assert cost == pytest.approx(expected) @@ -3362,7 +3488,9 @@ def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens }, ) - expected_prompt = (4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125 + expected_prompt = ( + (4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125 + ) expected_completion = 100 * 0.000015 assert prompt_cost == pytest.approx(expected_prompt) @@ -3402,7 +3530,10 @@ def test_extract_cache_read_tokens_zero_when_missing(): assert _extract_cache_read_tokens({}) == 0 assert _extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0 - assert _extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}}) == 0 + assert ( + _extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}}) + == 0 + ) def test_extract_cache_creation_tokens_anthropic_top_level(): @@ -3444,7 +3575,12 @@ def test_extract_cache_creation_tokens_zero_when_missing(): assert _extract_cache_creation_tokens({}) == 0 assert _extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0 - assert _extract_cache_creation_tokens({"prompt_tokens_details": {"cache_write_tokens": None}}) == 0 + assert ( + _extract_cache_creation_tokens( + {"prompt_tokens_details": {"cache_write_tokens": None}} + ) + == 0 + ) def test_custom_pricing_anthropic_style_cache_tokens_not_double_counted(): @@ -3571,6 +3707,7 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import Choices, CompletionTokensDetailsWrapper, Message + logging_obj = Logging( model="gemini-2.5-flash", messages=[{"role": "user", "content": "Hello"}], @@ -3597,8 +3734,12 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma prompt_tokens=209, completion_tokens=3996, total_tokens=4205, - completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=3114, text_tokens=882), - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, text_tokens=109), + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=3114, text_tokens=882 + ), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=100, text_tokens=109 + ), ), ) @@ -3664,7 +3805,9 @@ def test_completion_cost_logs_the_rates_it_billed_at(monkeypatch): assert rates is not None assert rates.input_cost_per_token == pytest.approx(6e-6) assert rates.cache_read_input_token_cost == pytest.approx(6e-7) - assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100_000 * rates.cache_read_input_token_cost) + assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx( + 100_000 * rates.cache_read_input_token_cost + ) assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(1_000 * rates.output_cost_per_token) @@ -3835,7 +3978,11 @@ def test_completion_cost_bills_interactions_api_response(): cost = completion_cost(completion_response=response, custom_llm_provider="gemini") reasoning_rate = model_info.get("output_cost_per_reasoning_token") or model_info["output_cost_per_token"] - expected = 100 * model_info["input_cost_per_token"] + 50 * model_info["output_cost_per_token"] + 25 * reasoning_rate + expected = ( + 100 * model_info["input_cost_per_token"] + + 50 * model_info["output_cost_per_token"] + + 25 * reasoning_rate + ) assert cost == pytest.approx(expected) assert cost > 0 @@ -4006,9 +4153,7 @@ def test_completion_cost_prices_anthropic_shaped_cache_read_tokens(_local_model_ assert cost == pytest.approx(3 * 4e-6 + 4014 * 4e-7 + 5 * 2e-5, rel=1e-9) -def _together_chat_response( - model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int -) -> ModelResponse: +def _together_chat_response(model: str, prompt_tokens: int, completion_tokens: int, cached_tokens: int) -> ModelResponse: return ModelResponse( id="chatcmpl-together-cache", choices=[{"finish_reason": "stop", "index": 0, "message": {"content": "acknowledged", "role": "assistant"}}], @@ -4076,8 +4221,6 @@ def test_completion_cost_together_metadata_only_model_still_uses_size_bucket(_lo ) assert cost == pytest.approx((23 + 15) * 8e-07, rel=1e-9) - - def test_select_model_name_strips_unregistered_alias_prefix(_local_model_cost_map): """A router-facing model_name alias containing "/" whose leading segment is NOT a registered provider must not be double-prefixed into a non-existent cost key. @@ -4318,7 +4461,9 @@ def test_every_one_hour_cache_write_rate_is_double_its_input_rate(): """Guard against pasting one model's 1h cache-write price onto another: every provider LiteLLM tracks (Anthropic, Bedrock, Vertex, Azure) publishes the 1h write at 2x input.""" - cost_map = json.loads((Path(__file__).parents[2] / "model_prices_and_context_window.json").read_text()) + cost_map = json.loads( + (Path(__file__).parents[2] / "model_prices_and_context_window.json").read_text() + ) one_hour_prefix = "cache_creation_input_token_cost_above_1hr" deviations = { (name, key): (entry["input_cost_per_token" + key[len(one_hour_prefix) :]], entry[key]) @@ -4524,7 +4669,9 @@ def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_m usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - prompt_cost, completion_cost = batch_cost_calculator(usage=usage, model="gpt-6-astra", custom_llm_provider="openai") + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, model="gpt-6-astra", custom_llm_provider="openai" + ) assert prompt_cost == pytest.approx(1000 * 5e-6) assert completion_cost == pytest.approx(500 * 2.5e-5) From 59c4cf94397207fa0604d58dac31027b94b8467f Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 12 Sep 2026 23:37:07 +0000 Subject: [PATCH 015/114] refactor(responses): build InputTokensDetails without post-construction mutation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index c75e5d0f5ea..14957a5e5aa 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2748,17 +2748,20 @@ class LiteLLMCompletionResponsesConfig: cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr( prompt_details, "cache_creation_tokens", None ) - input_tokens_details: Final = InputTokensDetails( + cache_write_extra: Final[Mapping[str, int]] = ( + MappingProxyType({"cache_write_tokens": cache_write_tokens}) + if cache_write_tokens is not None + else MappingProxyType({}) + ) + response_usage.input_tokens_details = InputTokensDetails( cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, text_tokens=prompt_details.text_tokens, audio_tokens=prompt_details.audio_tokens, cached_tokens_details=( cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None ), + **cache_write_extra, ) - if cache_write_tokens is not None: - setattr(input_tokens_details, "cache_write_tokens", cache_write_tokens) - response_usage.input_tokens_details = input_tokens_details # Translate completion_tokens_details to output_tokens_details if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None: From 4b84c83788a8e9e4db02b0b85a5f43fe0c49ea30 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:59:23 -0700 Subject: [PATCH 016/114] fix(cost): split the cache read breakdown at the audio cache-read rate --- .../litellm_core_utils/llm_cost_calc/utils.py | 32 +++++++++++++++---- .../llm_cost_calc/test_llm_cost_calc_utils.py | 23 +++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index dc689ca9618..8fc428b38ae 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1355,6 +1355,7 @@ class BilledTokenRates: input_cost_per_token: float output_cost_per_token: float cache_read_input_token_cost: float + cache_read_input_audio_token_cost: float cache_creation_input_token_cost: float cache_creation_input_token_cost_above_1hr: float output_cost_per_reasoning_token: float @@ -1366,6 +1367,7 @@ class BilledTokenRates: input_cost_per_token=self.input_cost_per_token * multiplier, output_cost_per_token=self.output_cost_per_token * multiplier, cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier, + cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier, cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, @@ -1389,15 +1391,16 @@ def _reasoning_token_count(usage: Usage) -> int: return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) -def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]: - """(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details - first, then the private top-level counters the Usage constructor mirrors cache tokens onto for - providers/callers that bypass the details.""" +def _cache_token_counts(usage: Usage) -> tuple[int, int, int, CacheCreationTokenDetails | None]: + """(cache read tokens, cached audio tokens, cache creation tokens, cache creation details): read from + prompt_tokens_details first, then the private top-level counters the Usage constructor mirrors cache + tokens onto for providers/callers that bypass the details.""" parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0 parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0 return ( parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)), + parsed["cache_hit_audio_tokens"] if parsed is not None else 0, parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)), parsed["cache_creation_token_details"] if parsed is not None else None, ) @@ -1408,11 +1411,13 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does.""" input_rate: Final = custom_cost_per_token["input_cost_per_token"] output_rate: Final = custom_cost_per_token["output_cost_per_token"] + cache_read_rate: Final = custom_cost_per_token.get("cache_read_input_token_cost", input_rate) cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate) return BilledTokenRates( input_cost_per_token=input_rate, output_cost_per_token=output_rate, - cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate), + cache_read_input_token_cost=cache_read_rate, + cache_read_input_audio_token_cost=cache_read_rate, cache_creation_input_token_cost=cache_creation_rate, cache_creation_input_token_cost_above_1hr=cache_creation_rate, output_cost_per_reasoning_token=output_rate, @@ -1449,6 +1454,11 @@ def _cost_map_billed_rates( completion_base_cost=completion_base_cost, current_time=billing_time, ) + audio_cache_read_rate: Final = _get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), + None, + ) multiplier: Final = ( _get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(model_info, vertex_location) @@ -1458,6 +1468,9 @@ def _cost_map_billed_rates( input_cost_per_token=prompt_base_cost, output_cost_per_token=completion_base_cost, cache_read_input_token_cost=cache_read_cost_rate, + cache_read_input_audio_token_cost=( + audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost_rate + ), cache_creation_input_token_cost=cache_creation_cost_rate, cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, output_cost_per_reasoning_token=reasoning_rate, @@ -1530,7 +1543,9 @@ def get_token_type_cost_breakdown( if rates is None: return TokenTypeCostBreakdown(0.0, 0.0, 0.0) - cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage) + cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( + usage + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1543,7 +1558,10 @@ def get_token_type_cost_breakdown( ) return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, - cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost, + cache_read_cost=( + float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + ), cache_creation_cost=cache_creation_cost, rates=rates, ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 5ff1ab62698..21b96127a74 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4006,6 +4006,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp input_cost_per_token=6e-6, output_cost_per_token=3e-5, cache_read_input_token_cost=6e-7, + cache_read_input_audio_token_cost=6e-7, cache_creation_input_token_cost=7.5e-6, cache_creation_input_token_cost_above_1hr=0.0, output_cost_per_reasoning_token=3e-5, @@ -5252,3 +5253,25 @@ def test_cached_audio_tokens_billed_at_audio_cache_rate_through_model_info_looku prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai") assert prompt_cost == pytest.approx(300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7) + + +def test_cache_read_breakdown_splits_cached_audio_at_the_audio_cache_rate(_local_model_cost_map: None) -> None: + usage = Usage( + prompt_tokens=4863, + completion_tokens=1087, + total_tokens=5950, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=1693, + audio_tokens=3170, + cached_tokens=2816, + cached_tokens_details={"text_tokens": 896, "audio_tokens": 1920}, + ), + ) + + breakdown = get_token_type_cost_breakdown(model="gpt-realtime-2.1-mini", custom_llm_provider="openai", usage=usage) + prompt_cost, _ = generic_cost_per_token(model="gpt-realtime-2.1-mini", usage=usage, custom_llm_provider="openai") + + assert breakdown.cache_read_cost == pytest.approx(896 * 6e-8 + 1920 * 3e-7) + assert breakdown.rates is not None + assert breakdown.rates.cache_read_input_audio_token_cost == pytest.approx(3e-7) + assert prompt_cost == pytest.approx((1693 - 896) * 6e-7 + (3170 - 1920) * 1e-5 + breakdown.cache_read_cost) From 8577d63ff5212173aa5b30bdfe8118b666c74179 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 00:14:49 +0000 Subject: [PATCH 017/114] fix(proxy): forward provider request id headers on mapped error responses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 10 +++--- .../proxy/test_common_request_processing.py | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e6ed60ba177..9b8a98ca0e7 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3445,15 +3445,13 @@ class ProxyBaseLLMRequestProcessing: # a failed request reports no timing, matching /v1/chat/completions read_timing_from_logging_obj=False, ) - # Extract headers from exception - check both e.headers and e.response.headers headers = getattr(e, "headers", None) or {} if not headers: - # Try to get headers from e.response.headers (httpx.Response) _response: Final = attribute_of(e, "response") - if _response is not None: - _response_headers: Final = getattr(_response, "headers", None) - if _response_headers: - headers = get_response_headers(dict(_response_headers)) + _response_headers: Final = getattr(_response, "headers", None) if _response is not None else None + _provider_headers: Final = _response_headers or getattr(e, "litellm_response_headers", None) + if _provider_headers: + headers = get_response_headers(dict(_provider_headers)) headers.update(custom_headers) # Call response headers hook for failure diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index efbb5eedad4..45e436f756a 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -8386,6 +8386,41 @@ async def test_handle_llm_api_exception_forwards_provider_headers_on_http_status assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-passthrough-500" +@pytest.mark.asyncio +async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_response_is_synthetic(): + """Exception mapping hands the proxy a mapped error whose ``response`` is a synthetic empty + ``httpx.Response`` and parks the provider's real headers on ``litellm_response_headers``. + The client must still get the provider request id, as it does on a 200. + """ + import httpx + + from litellm.proxy._types import ProxyException, UserAPIKeyAuth + + mapped = litellm.BadRequestError( + message="OpenAIException - max_tokens is too large: 999999999.", + model="gpt-4o-mini", + llm_provider="openai", + ) + mapped.litellm_response_headers = httpx.Headers({"x-request-id": "req_openai_400"}) + assert dict(mapped.response.headers) == {} + + processor = ProxyBaseLLMRequestProcessing(data={}) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with pytest.raises(ProxyException) as exc_info: + await processor._handle_llm_api_exception( + e=mapped, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert exc_info.value.code == "400" + assert "max_tokens is too large: 999999999." in exc_info.value.message + assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" + + class TestBackgroundResponseRetrievalGovernance: """LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines.""" From f41c8556b5e22d1cd980df9ba2df7c0b8336d545 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 00:05:50 +0000 Subject: [PATCH 018/114] feat(jwt): allow virtual_key_claim_field per issuer Multi-IdP deployments can now set virtual_key_claim_field and unregistered_jwt_client_behavior on a JWTIssuerConfig entry. Tokens from that issuer use the issuer-specific claim path and no-match policy for the virtual key mapping lookup; issuers that omit them keep the global values. The auth flow now enters the mapping lookup when any issuer configures the field, not only when the global field is set. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 30 +++ litellm/proxy/auth/user_api_key_auth.py | 19 +- .../proxy/auth/test_user_api_key_auth.py | 225 +++++++++++++++++- tests/test_litellm/proxy/test__types.py | 58 +++++ 4 files changed, 324 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ae6c042ab3a..3ede847370d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4837,6 +4837,14 @@ class JWTIssuerConfig(BaseModel): default=None, description="Issuer-specific claim path to normalize into LiteLLM's end-user id.", ) + virtual_key_claim_field: str | None = Field( + default=None, + description="Issuer-specific claim path used for the virtual key mapping lookup. Falls back to the global field.", + ) + unregistered_jwt_client_behavior: UnregisteredJWTClientBehavior | None = Field( + default=None, + description="Issuer-specific policy when the virtual key claim has no mapping. Falls back to the global policy.", + ) model_config = { "extra": "forbid", @@ -5063,6 +5071,28 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): super().__init__(**kwargs) + def get_issuer_config(self, issuer: str | None) -> JWTIssuerConfig | None: + if issuer is None or self.issuers is None: + return None + return next((config for config in self.issuers if config.issuer == issuer), None) + + def is_virtual_key_mapping_configured(self) -> bool: + if self.virtual_key_claim_field is not None: + return True + return any(config.virtual_key_claim_field is not None for config in self.issuers or ()) + + def get_virtual_key_claim_field(self, issuer: str | None) -> str | None: + issuer_config: Final = self.get_issuer_config(issuer) + if issuer_config is not None and issuer_config.virtual_key_claim_field is not None: + return issuer_config.virtual_key_claim_field + return self.virtual_key_claim_field + + def get_unregistered_jwt_client_behavior(self, issuer: str | None) -> UnregisteredJWTClientBehavior: + issuer_config: Final = self.get_issuer_config(issuer) + if issuer_config is not None and issuer_config.unregistered_jwt_client_behavior is not None: + return issuer_config.unregistered_jwt_client_behavior + return self.unregistered_jwt_client_behavior + class PrismaCompatibleUpdateDBModel(TypedDict, total=False): model_name: str diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 9828311112e..f1b373da898 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -987,9 +987,12 @@ async def _resolve_jwt_to_virtual_key( - Raises HTTPException: REJECT policy hit, missing claim under REJECT/AUTO_REGISTER, or other policy violations. """ - virtual_key_claim_field: Final = jwt_handler.litellm_jwtauth.virtual_key_claim_field + raw_issuer: Final = jwt_claims.get(JWTHandler.LITELLM_JWT_ISSUER_CLAIM) + normalized_issuer: Final = raw_issuer if isinstance(raw_issuer, str) else None + virtual_key_claim_field: Final = jwt_handler.litellm_jwtauth.get_virtual_key_claim_field(normalized_issuer) if virtual_key_claim_field is None: return None + behavior: Final = jwt_handler.litellm_jwtauth.get_unregistered_jwt_client_behavior(normalized_issuer) claim_value: Final = get_nested_value( data=jwt_claims, @@ -1006,7 +1009,6 @@ async def _resolve_jwt_to_virtual_key( # simply by presenting a JWT that omits the configured field. For # AUTO_REGISTER there is no stable identity to map without a claim # value, so we deny rather than create a sentinel-keyed record. - behavior = jwt_handler.litellm_jwtauth.unregistered_jwt_client_behavior if behavior in ( UnregisteredJWTClientBehavior.REJECT, UnregisteredJWTClientBehavior.AUTO_REGISTER, @@ -1021,7 +1023,13 @@ async def _resolve_jwt_to_virtual_key( return None cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value)) - cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key) + raw_cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key) + sentinel_written_by_this_policy: Final = behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER + cached_mapping: Final = ( + None + if raw_cached_mapping == _JWT_PROXY_ADMIN_SENTINEL and not sentinel_written_by_this_policy + else raw_cached_mapping + ) if cached_mapping == _JWT_PROXY_ADMIN_SENTINEL: # Previously resolved to a proxy admin via auth_builder; skip the @@ -1030,7 +1038,6 @@ async def _resolve_jwt_to_virtual_key( return None if cached_mapping == "__NO_MAPPING__": - behavior = jwt_handler.litellm_jwtauth.unregistered_jwt_client_behavior if behavior == UnregisteredJWTClientBehavior.REJECT: raise HTTPException( status_code=403, @@ -1093,8 +1100,6 @@ async def _resolve_jwt_to_virtual_key( ) # No mapping found (DB miss or no DB) — apply no-match policy. - behavior = jwt_handler.litellm_jwtauth.unregistered_jwt_client_behavior - if behavior == UnregisteredJWTClientBehavior.REJECT: # Cache the miss before raising so repeated rejections are served from # cache and don't re-query the DB on every request. @@ -1428,7 +1433,7 @@ async def _user_api_key_auth_builder( # unnecessary DB queries in auth_builder do_standard_jwt_auth = True pending_auto_register: _PendingAutoRegister | None = None - if jwt_handler.litellm_jwtauth.virtual_key_claim_field is not None: + if jwt_handler.litellm_jwtauth.is_virtual_key_mapping_configured(): # Decode JWT to get claims without running full auth_builder jwt_claims: dict | None if jwt_handler.litellm_jwtauth.oidc_userinfo_enabled and not is_jwt: diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 0cdbcde6abc..dac6b7aedc5 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -13,7 +13,7 @@ from unittest.mock import ANY, AsyncMock, MagicMock, patch import pytest -from fastapi import status +from fastapi import HTTPException, status import litellm import litellm.proxy.proxy_server @@ -7442,3 +7442,226 @@ async def test_claude_view_never_reinterprets_explicit_names(monkeypatch, layer) assert data["model"] == ("foo" if layer == "unclaimed" else encoded) await _normalize_claude_model(data, token, request, "/v1/messages") assert data["model"] == ("foo" if layer == "unclaimed" else encoded) + + +ISSUER_ONE = "https://issuer-one.example.com" +ISSUER_TWO = "https://issuer-two.example.com" + + +def _per_issuer_virtual_key_jwt_handler( + global_claim_field: str | None, global_behavior: str = "fallback_team_mapping" +) -> MagicMock: + jwt_handler = MagicMock() + jwt_handler.is_jwt.return_value = True + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth( + virtual_key_claim_field=global_claim_field, + unregistered_jwt_client_behavior=global_behavior, + issuers=[ + { + "issuer": ISSUER_ONE, + "jwks_url": f"{ISSUER_ONE}/keys", + "audience": "audience-one", + "team_id_jwt_field": "sub", + }, + { + "issuer": ISSUER_TWO, + "jwks_url": f"{ISSUER_TWO}/keys", + "audience": "audience-two", + "virtual_key_claim_field": "sub", + "unregistered_jwt_client_behavior": "reject", + }, + ], + ) + return jwt_handler + + +def _fake_prisma_with_jwt_key_mapping(hashed_token: str | None) -> tuple[SimpleNamespace, AsyncMock]: + find_first = AsyncMock(return_value=None if hashed_token is None else SimpleNamespace(token=hashed_token)) + prisma_client = SimpleNamespace(db=SimpleNamespace(litellm_jwtkeymapping=SimpleNamespace(find_first=find_first))) + return prisma_client, find_first + + +def _mapping_where(claim_name: str, claim_value: str) -> dict[str, str | bool]: + return {"jwt_claim_name": claim_name, "jwt_claim_value": claim_value, "is_active": True} + + +@pytest.mark.asyncio +async def test_per_issuer_virtual_key_claim_field_selects_the_issuer_mapping_for_the_db_lookup(): + from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key + + jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field=None) + prisma_client, find_first = _fake_prisma_with_jwt_key_mapping("hashed-mapped-key") + user_api_key_cache = DualCache() + await user_api_key_cache.async_set_cache( + key="hashed-mapped-key", + value=UserAPIKeyAuth(token="hashed-mapped-key", api_key="hashed-mapped-key", team_id="svc-team"), + ) + + resolved = await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "svc-account-7"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + find_first.assert_awaited_once_with(where=_mapping_where("sub", "svc-account-7")) + assert isinstance(resolved, UserAPIKeyAuth) + assert resolved.token == "hashed-mapped-key" + assert resolved.team_id == "svc-team" + assert await user_api_key_cache.async_get_cache("jwt_key_mapping:sub:svc-account-7") == "hashed-mapped-key" + + +@pytest.mark.asyncio +async def test_per_issuer_reject_behavior_does_not_leak_into_the_team_issuer(): + from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key + + jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field=None) + prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None) + + team_issuer_result = await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "team-alpha"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + assert team_issuer_result is None + find_first.assert_not_awaited() + + with pytest.raises(HTTPException) as exc: + await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "unknown-svc"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + assert exc.value.status_code == 403 + assert "No registered mapping for sub='unknown-svc'" in str(exc.value.detail) + find_first.assert_awaited_once_with(where=_mapping_where("sub", "unknown-svc")) + + +@pytest.mark.asyncio +async def test_proxy_admin_sentinel_cached_by_another_issuer_does_not_bypass_reject(): + from litellm.proxy.auth.user_api_key_auth import _JWT_PROXY_ADMIN_SENTINEL, _resolve_jwt_to_virtual_key + + jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="sub", global_behavior="auto_register") + prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None) + user_api_key_cache = DualCache() + await user_api_key_cache.async_set_cache(key="jwt_key_mapping:sub:admin-7", value=_JWT_PROXY_ADMIN_SENTINEL) + + auto_register_issuer_result = await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "admin-7"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + assert auto_register_issuer_result is None + find_first.assert_not_awaited() + + with pytest.raises(HTTPException) as exc: + await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "admin-7"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + assert exc.value.status_code == 403 + assert "No registered mapping for sub='admin-7'" in str(exc.value.detail) + find_first.assert_awaited_once_with(where=_mapping_where("sub", "admin-7")) + + +@pytest.mark.asyncio +async def test_issuer_without_virtual_key_claim_field_falls_back_to_the_global_field(): + from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key + + jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="client_id") + prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None) + + with_claim = await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "team-alpha", "client_id": "app-9"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + without_claim = await _resolve_jwt_to_virtual_key( + jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "team-alpha"}, + jwt_handler=jwt_handler, + prisma_client=prisma_client, + user_api_key_cache=DualCache(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + assert with_claim is None + assert without_claim is None + find_first.assert_awaited_once_with(where=_mapping_where("client_id", "app-9")) + + +@pytest.mark.asyncio +async def test_auth_flow_enters_virtual_key_mapping_when_only_an_issuer_configures_the_claim_field(): + jwt_token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJzdmMtYWNjb3VudC03In0.signature" + jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field=None) + jwt_handler.auth_jwt = AsyncMock( + return_value={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "svc-account-7"} + ) + mapped_key = UserAPIKeyAuth(token="hashed-mapped-key", api_key="hashed-mapped-key", team_id="svc-team") + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.method = "POST" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + mock_request.state = SimpleNamespace() + + with ( + patch( # test-quality-ok: the builder reads proxy settings from module globals, no injection seam + "litellm.proxy.proxy_server.general_settings", {"enable_jwt_auth": True} + ), + patch("litellm.proxy.proxy_server.premium_user", True), # test-quality-ok: module-global proxy state + patch("litellm.proxy.proxy_server.master_key", "sk-master"), # test-quality-ok: module-global proxy state + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: module-global proxy state + patch( # test-quality-ok: module-global proxy state + "litellm.proxy.proxy_server.user_api_key_cache", DualCache() + ), + patch( # test-quality-ok: module-global proxy state + "litellm.proxy.proxy_server.proxy_logging_obj", MagicMock() + ), + patch("litellm.proxy.proxy_server.jwt_handler", jwt_handler), # test-quality-ok: module-global proxy state + patch( # test-quality-ok: the regression is whether the builder reaches this seam at all + "litellm.proxy.auth.user_api_key_auth._resolve_jwt_to_virtual_key", + new_callable=AsyncMock, + return_value=mapped_key, + ) as resolve_mock, + patch( # test-quality-ok: a mapped key must short-circuit standard JWT auth; reaching it is the failure + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + side_effect=AssertionError("standard JWT auth must not run for a mapped virtual key"), + ), + ): + result = await _user_api_key_auth_builder( + request=mock_request, + api_key=jwt_token, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={"model": "gpt-4o-mini"}, + ) + + resolve_mock.assert_awaited_once() + assert resolve_mock.await_args.kwargs["jwt_claims"][JWTHandler.LITELLM_JWT_ISSUER_CLAIM] == ISSUER_TWO + assert result.api_key == "hashed-mapped-key" + assert result.team_id == "svc-team" diff --git a/tests/test_litellm/proxy/test__types.py b/tests/test_litellm/proxy/test__types.py index 26bb1533da4..9e1486ce90f 100644 --- a/tests/test_litellm/proxy/test__types.py +++ b/tests/test_litellm/proxy/test__types.py @@ -277,3 +277,61 @@ def test_team_membership_budget_table_present_still_works(): } result = LiteLLM_TeamMembership.model_validate(data) assert result.litellm_budget_table is None + + +def test_a_jwt_issuer_can_override_the_virtual_key_claim_field_while_other_issuers_keep_the_global_one(): + from litellm.proxy._types import LiteLLM_JWTAuth, UnregisteredJWTClientBehavior + + jwt_auth = LiteLLM_JWTAuth( + virtual_key_claim_field="client_id", + issuers=[ + { + "issuer": "https://team-idp.example.com", + "jwks_url": "https://team-idp.example.com/keys", + "audience": "litellm", + "team_id_jwt_field": "sub", + }, + { + "issuer": "https://service-idp.example.com", + "jwks_url": "https://service-idp.example.com/keys", + "audience": "litellm", + "virtual_key_claim_field": "sub", + "unregistered_jwt_client_behavior": "reject", + }, + ], + ) + + assert jwt_auth.get_virtual_key_claim_field("https://service-idp.example.com") == "sub" + assert jwt_auth.get_unregistered_jwt_client_behavior("https://service-idp.example.com") is ( + UnregisteredJWTClientBehavior.REJECT + ) + assert jwt_auth.get_virtual_key_claim_field("https://team-idp.example.com") == "client_id" + assert jwt_auth.get_unregistered_jwt_client_behavior("https://team-idp.example.com") is ( + UnregisteredJWTClientBehavior.FALLBACK_TEAM_MAPPING + ) + assert jwt_auth.get_virtual_key_claim_field(None) == "client_id" + assert jwt_auth.get_virtual_key_claim_field("https://unknown-idp.example.com") == "client_id" + + +@pytest.mark.parametrize( + ("global_field", "issuer_field", "is_configured"), + ((None, None, False), ("sub", None, True), (None, "sub", True)), +) +def test_virtual_key_mapping_counts_as_configured_when_any_issuer_sets_the_claim_field( + global_field, issuer_field, is_configured +): + from litellm.proxy._types import LiteLLM_JWTAuth + + jwt_auth = LiteLLM_JWTAuth( + virtual_key_claim_field=global_field, + issuers=[ + { + "issuer": "https://idp.example.com", + "jwks_url": "https://idp.example.com/keys", + "audience": "litellm", + "virtual_key_claim_field": issuer_field, + } + ], + ) + + assert jwt_auth.is_virtual_key_mapping_configured() is is_configured From a94c060b84815438dc3a6dda379e0fb7fa60448f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 12 Sep 2026 17:48:00 -0700 Subject: [PATCH 019/114] fix(cost): fill the missing realtime cache-read rates --- ...odel_prices_and_context_window_backup.json | 9 ++++-- model_prices_and_context_window.json | 9 ++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 28 +++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index de9867f9eee..2ce8914ba5a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5513,7 +5513,8 @@ }, "azure/gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5546,7 +5547,8 @@ }, "azure/gpt-realtime-1.5-2026-02-23": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5683,6 +5685,7 @@ }, "azure/gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -5715,6 +5718,7 @@ }, "azure/gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -32683,6 +32687,7 @@ "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index de9867f9eee..2ce8914ba5a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5513,7 +5513,8 @@ }, "azure/gpt-realtime-2025-08-28": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-03-02", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5546,7 +5547,8 @@ }, "azure/gpt-realtime-1.5-2026-02-23": { "cache_creation_input_audio_token_cost": 4e-06, - "cache_read_input_token_cost": 4e-06, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-08-24", "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -5683,6 +5685,7 @@ }, "azure/gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -5715,6 +5718,7 @@ }, "azure/gpt-realtime-mini-2025-10-06": { "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -32683,6 +32687,7 @@ "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b96127a74..7cd19c65887 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -5275,3 +5275,31 @@ def test_cache_read_breakdown_splits_cached_audio_at_the_audio_cache_rate(_local assert breakdown.rates is not None assert breakdown.rates.cache_read_input_audio_token_cost == pytest.approx(3e-7) assert prompt_cost == pytest.approx((1693 - 896) * 6e-7 + (3170 - 1920) * 1e-5 + breakdown.cache_read_cost) + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_prompt_cost"), + ( + pytest.param("azure/gpt-realtime-2025-08-28", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime"), + pytest.param("azure/gpt-realtime-1.5-2026-02-23", "azure", 300 * 4e-6 + 100 * 4e-7 + 200 * 3.2e-5 + 400 * 4e-7, id="azure-gpt-realtime-1.5"), + pytest.param("azure/gpt-realtime-mini", "azure", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="azure-gpt-realtime-mini"), + pytest.param("gpt-realtime-mini", "openai", 300 * 6e-7 + 100 * 6e-8 + 200 * 1e-5 + 400 * 3e-7, id="openai-gpt-realtime-mini"), + ), +) +def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates( + _local_model_cost_map: None, model: str, custom_llm_provider: str, expected_prompt_cost: float +) -> None: + usage = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=600, + cached_tokens=500, + cached_tokens_details={"text_tokens": 100, "audio_tokens": 400}, + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider) + assert prompt_cost == pytest.approx(expected_prompt_cost) From 555e321cf170ad2d15a2932fccdea104da1c39b6 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 01:05:51 +0000 Subject: [PATCH 020/114] fix(router): record flat retry attempts and cap retries from attempted_retries Router.log_retry used to copy the failed attempt's kwargs and metadata into metadata.previous_models. Nothing downstream read those copies, but they carried client credentials into spend logs and grew the payload on every retry. Each attempt now leaves a flat record (model group, deployment id, exception type and string, attempt number), which drops RETRY_BREADCRUMB_EXCLUDED_KWARGS and the per-retry credential masking. num_retries_per_request was enforced from len(previous_models), which only looked at the metadata bucket and never exceeded four records. The sync and async client wrappers and the Rust lifecycle guard now read attempted_retries from whichever metadata bucket the call carries. Resolves LIT-7505 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 2 +- litellm/litellm_core_utils/core_helpers.py | 13 +++ litellm/router.py | 45 +++----- litellm/rust_bridge/lifecycle.py | 16 +-- litellm/types/router.py | 8 ++ litellm/utils.py | 18 +-- .../test_router_helper_utils.py | 29 +++-- .../rust_bridge/test_lifecycle.py | 30 +++++ tests/test_litellm/test_router.py | 106 +++++++++++------- tests/test_litellm/test_utils.py | 47 ++++++++ tests/test_litellm_rust/ocr/test_lifecycle.py | 2 +- 11 files changed, 209 insertions(+), 107 deletions(-) create mode 100644 tests/test_litellm/rust_bridge/test_lifecycle.py diff --git a/litellm/__init__.py b/litellm/__init__.py index ccfbf80369f..261457d6889 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -538,7 +538,7 @@ context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 allow_dynamic_callback_disabling: bool = True -num_retries_per_request: Optional[int] = None # for the request overall (incl. fallbacks + model retries) +num_retries_per_request: Optional[int] = None # cap on Router retries of one model group; resets per fallback hop ####### SECRET MANAGERS ##################### secret_manager_client: Optional[Any] = ( None # list of instantiated key management clients - e.g. azure kv, infisical, etc. diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index aa7d6ca1699..ecd9cdac88b 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -303,6 +303,19 @@ def get_metadata_variable_name_from_kwargs( return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" +def max_retries_per_request_hit(kwargs: Mapping[str, object], num_retries_per_request: int | None) -> bool: + """ + Whether the Router retry about to run (``attempted_retries`` >= 1 in the metadata bucket) is past the cap + """ + if num_retries_per_request is None: + return False + metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs)) + if not isinstance(metadata, Mapping): + return False + attempted_retries: Final = metadata.get("attempted_retries") + return type(attempted_retries) is int and 0 < attempted_retries and num_retries_per_request <= attempted_retries + + def get_or_create_metadata_bucket( request_data: dict, ) -> tuple[Literal["metadata", "litellm_metadata"], dict]: diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..a46ffa83b05 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -96,7 +96,6 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, - mask_credentials_in_payload, mask_sensitive_structure, ) from litellm.litellm_core_utils.token_counter import offload_token_count @@ -242,6 +241,7 @@ from litellm.types.router import ( ModelGroupInfo, OptionalPreCallChecks, PreRoutingStrategy, + RetryAttemptRecord, RetryPolicy, RouterCacheEnum, RouterErrors, @@ -623,20 +623,6 @@ def _replay_live_router_model_cost() -> None: set_live_deployment_replay(_replay_live_router_model_cost) -# Kwargs that carry no signal about the failed attempt, so log_retry drops them from a -# breadcrumb entirely: the request payload, the proxy's snapshot of the inbound request (its body -# aliases the live request metadata, earlier breadcrumbs included, so copying it would nest every -# breadcrumb inside the next one), and the router-internal walk state. Credentials are handled -# separately by mask_credentials_in_payload, which scrubs credential-named values from whatever -# kwargs remain rather than trying to enumerate every credential-bearing key here. -RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( - ( - "messages", - "original_function", - "attempted_targets", - "proxy_server_request", - ) -) RETRY_BREADCRUMB_LIMIT: Final = 4 @@ -8374,31 +8360,28 @@ class Router: def log_retry(self, kwargs: dict, e: Exception) -> dict: """ - When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing + When a retry or fallback happens, record which model group, deployment and attempt just failed and why """ _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var] - attempt_kwargs: Final = MappingProxyType( - {k: v for k, v in kwargs.items() if k != _metadata_var and k not in RETRY_BREADCRUMB_EXCLUDED_KWARGS} - ) - attempt_metadata: Final = MappingProxyType( - {k: v for k, v in request_metadata.items() if k != "previous_models"} - ) - previous_model: Final = MappingProxyType( - { - "exception_type": type(e).__name__, - "exception_string": str(e), - **attempt_kwargs, - _metadata_var: attempt_metadata, - } - ) + model_group: Final = kwargs.get("model") + model_info: Final = request_metadata.get("model_info") + deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + attempted_retries: Final = request_metadata.get("attempted_retries") + attempt_record: Final[RetryAttemptRecord] = { + "model_group": model_group if isinstance(model_group, str) else None, + "deployment_id": deployment_id if isinstance(deployment_id, str) else None, + "exception_type": type(e).__name__, + "exception_string": str(e), + "attempted_retries": attempted_retries if type(attempted_retries) is int else None, + } earlier_breadcrumbs: Final = request_metadata.get("previous_models") kept_breadcrumbs: Final[tuple[object, ...]] = ( tuple(earlier_breadcrumbs)[-(RETRY_BREADCRUMB_LIMIT - 1) :] if isinstance(earlier_breadcrumbs, (list, tuple)) else () ) - breadcrumbs: Final = (*kept_breadcrumbs, mask_credentials_in_payload(previous_model)) + breadcrumbs: Final = (*kept_breadcrumbs, attempt_record) kwargs[_metadata_var]["previous_models"] = breadcrumbs # rebind-ok: the logging object already holds this dict return kwargs diff --git a/litellm/rust_bridge/lifecycle.py b/litellm/rust_bridge/lifecycle.py index f5e0c1b0fc6..f1cc912129d 100644 --- a/litellm/rust_bridge/lifecycle.py +++ b/litellm/rust_bridge/lifecycle.py @@ -99,23 +99,13 @@ def setup( def check_limits(kwargs: Mapping[str, object]) -> None: import litellm + from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit current_cost: Final = litellm._current_cost # pyright: ignore[reportPrivateUsage] # shared SDK budget counter has no public accessor if litellm.max_budget and current_cost > litellm.max_budget: raise litellm.BudgetExceededError(current_cost=current_cost, max_budget=litellm.max_budget) - metadata: Final = kwargs.get("metadata") - if isinstance(metadata, Mapping): - typed_metadata: Final = cast( # cast-ok: runtime Mapping check establishes read-only metadata - Mapping[str, object], metadata - ) - previous: Final = typed_metadata.get("previous_models") - if ( - isinstance(previous, list) - and litellm.num_retries_per_request is not None - and len(cast(list[object], previous)) # cast-ok: runtime list check establishes the retry history - >= litellm.num_retries_per_request - ): - raise RuntimeError("Max retries per request hit!") + if max_retries_per_request_hit(kwargs, litellm.num_retries_per_request): + raise RuntimeError("Max retries per request hit!") def finalize( diff --git a/litellm/types/router.py b/litellm/types/router.py index fc09c40fe08..fecc0e00f99 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -883,6 +883,14 @@ class RouterModelGroupAliasItem(TypedDict): hidden: bool # if 'True', don't return on `.get_model_list` +class RetryAttemptRecord(TypedDict): + model_group: ReadOnly[str | None] + deployment_id: ReadOnly[str | None] + exception_type: ReadOnly[str] + exception_string: ReadOnly[str] + attempted_retries: ReadOnly[int | None] + + VALID_LITELLM_ENVIRONMENTS = [ "development", "staging", diff --git a/litellm/utils.py b/litellm/utils.py index 394ab4b4094..98881e68986 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -81,7 +81,7 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit, normalize_drop_params from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, ) @@ -1509,12 +1509,8 @@ def client(original_function): call_type = original_function.__name__ if _is_async_request(kwargs): # [OPTIONAL] CHECK MAX RETRIES / REQUEST - if litellm.num_retries_per_request is not None: - # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) - if previous_models is not None: - if litellm.num_retries_per_request <= len(previous_models): - raise Exception("Max retries per request hit!") + if max_retries_per_request_hit(kwargs, litellm.num_retries_per_request): + raise Exception("Max retries per request hit!") # MODEL CALL result = original_function(*args, **kwargs) @@ -1573,12 +1569,8 @@ def client(original_function): ) # [OPTIONAL] CHECK MAX RETRIES / REQUEST - if litellm.num_retries_per_request is not None: - # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] - previous_models = (kwargs.get("metadata") or {}).get("previous_models", None) - if previous_models is not None: - if litellm.num_retries_per_request <= len(previous_models): - raise Exception("Max retries per request hit!") + if max_retries_per_request_hit(kwargs, litellm.num_retries_per_request): + raise Exception("Max retries per request hit!") # [OPTIONAL] CHECK CACHE print_verbose( diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 7dbac243d55..5b06c5fdb01 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1,3 +1,4 @@ +import json import os import traceback from dotenv import load_dotenv @@ -628,17 +629,29 @@ def test_deployment_callback_respects_cooldown_time(model_list): assert mock_set.call_args.kwargs["time_to_cooldown"] == 0 -def test_log_retry(model_list): - """Test if the '_log_retry' function is working correctly""" - import time - +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_log_retry(model_list, metadata_key): + """log_retry appends one flat record per failed attempt and copies neither the request kwargs nor + the request metadata into it""" router = Router(model_list=model_list) new_kwargs = router.log_retry( - kwargs={"metadata": {}}, - e=Exception(), + kwargs={ + "model": "gpt-3.5-turbo", + "api_key": "sk-must-not-be-recorded", + "messages": [{"role": "user", "content": "hi"}], + metadata_key: {"model_info": {"id": "deployment-1"}, "attempted_retries": 2, "user_api_key": "sk-proxy"}, + }, + e=litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-3.5-turbo"), ) - assert "metadata" in new_kwargs - assert "previous_models" in new_kwargs["metadata"] + assert json.loads(json.dumps(new_kwargs[metadata_key]["previous_models"])) == [ + { + "model_group": "gpt-3.5-turbo", + "deployment_id": "deployment-1", + "exception_type": "RateLimitError", + "exception_string": "litellm.RateLimitError: slow down", + "attempted_retries": 2, + } + ] def test_update_usage(model_list): diff --git a/tests/test_litellm/rust_bridge/test_lifecycle.py b/tests/test_litellm/rust_bridge/test_lifecycle.py new file mode 100644 index 00000000000..1f0b5591c2b --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_lifecycle.py @@ -0,0 +1,30 @@ +from typing import Final + +import pytest + +import litellm +from litellm.rust_bridge.lifecycle import check_limits + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize( + "cap, attempted_retries, refused", + [(5, 5, True), (5, 4, False), (0, 0, False), (0, 1, True)], + ids=[ + "cap-above-four-reached", + "cap-above-four-not-reached", + "first-attempt-passes-cap-of-zero", + "cap-of-zero-refuses-first-retry", + ], +) +def test_check_limits_reads_attempted_retries( + monkeypatch: pytest.MonkeyPatch, metadata_key: str, cap: int, attempted_retries: int, refused: bool +) -> None: + monkeypatch.setattr(litellm, "num_retries_per_request", cap) + monkeypatch.setattr(litellm, "max_budget", None) + kwargs: Final = {"model": "mistral/mistral-ocr-latest", metadata_key: {"attempted_retries": attempted_retries}} + if refused: + with pytest.raises(RuntimeError, match="Max retries per request hit!"): + check_limits(kwargs) + else: + check_limits(kwargs) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index f5e9b2091a0..6150ce287ed 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10607,6 +10607,7 @@ def _cyclic_fallback_router(num_retries=0): "api_key": "sk-fake", "mock_response": "litellm.InternalServerError", }, + "model_info": {"id": f"{group}-deployment"}, } for group in groups ], @@ -10656,28 +10657,37 @@ async def test_cyclic_fallback_graph_does_not_amplify_one_request(): assert sum(len(message) for message in capture.messages) < 5_000 +_FLAT_ATTEMPT_RECORD_KEYS = frozenset( + {"model_group", "deployment_id", "exception_type", "exception_string", "attempted_retries"} +) +_BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doNotShip" + + @pytest.mark.asyncio -async def test_retry_breadcrumbs_do_not_carry_the_walk_state(): - """log_retry copies every kwarg into previous_models, which reaches spend logs and - logging callbacks. The set of already-attempted groups is router-internal walk state - with no diagnostic value there, and it is the one entry that is not a plain scalar. - A retry has to be configured for the walk state to reach log_retry at all.""" +async def test_retry_records_are_flat_and_name_the_failed_group_on_fallback_hops(): + """Each failed attempt leaves a flat record in previous_models, which reaches spend logs and + logging callbacks. Nothing downstream reads the failed attempt's kwargs or metadata, and copying + them is what carried client credentials and multiplied the payload on every retry. A fallback hop + calls log_retry too, so the record has to name the group that failed, not the one taken next.""" router = _cyclic_fallback_router(num_retries=1) capture = _LogCapture(logging.ERROR) recorder = _FallbackAttemptRecorder() await _drive_cyclic_fallback(router, capture, recorder) - breadcrumbs = [breadcrumb for hop in recorder.breadcrumbs_per_target for breadcrumb in hop] - assert breadcrumbs, "no retry breadcrumbs were recorded" - assert any( - "fallback_depth" in breadcrumb for breadcrumb in breadcrumbs - ), "no breadcrumb carried router walk state, so this test cannot see the leak" - for breadcrumb in breadcrumbs: - assert "attempted_targets" not in breadcrumb - - -_BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doNotShip" + records = [record for hop in recorder.breadcrumbs_per_target for record in hop] + assert records, "no retry records were recorded" + for record in records: + assert set(record) == _FLAT_ATTEMPT_RECORD_KEYS + assert record["exception_type"] == "InternalServerError" + assert record["deployment_id"] == f"{record['model_group']}-deployment" + group_failed_before_hop = {"group-b": "group-a", "group-c": "group-b", "group-d": "group-c"} + for failed_target, hop_records in zip(recorder.failed_targets, recorder.breadcrumbs_per_target): + groups = [record["model_group"] for record in hop_records] + first_own_attempt = groups.index(failed_target) + assert groups[first_own_attempt - 1] == group_failed_before_hop[failed_target] + assert set(groups[first_own_attempt:]) == {failed_target} + assert [record["attempted_retries"] for record in hop_records[first_own_attempt:]][:2] == [0, 1] @pytest.mark.parametrize( @@ -10703,22 +10713,20 @@ _BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doN ], ) @pytest.mark.asyncio -async def test_retry_breadcrumbs_never_carry_a_forwarded_credential(container_key, request_kwargs): - """log_retry copies kwargs into previous_models, which reaches spend logs and logging callbacks. - Any of these kwargs can carry a client's forwarded Authorization token or a provider key, and a - breadcrumb has no diagnostic use for the raw secret. A denylist of key names is always one new - credential kwarg behind, so log_retry scrubs credential-named values by pattern instead: the - container still reaches the breadcrumb, but the raw secret never does, whatever key holds it.""" +async def test_retry_records_never_carry_a_forwarded_credential(container_key, request_kwargs): + """previous_models reaches spend logs and logging callbacks. Any request kwarg can carry a client's + forwarded Authorization token or a provider key, so the record must not carry request kwargs at + all: neither the credential-bearing container nor the raw secret, whatever key holds it.""" router = _cyclic_fallback_router(num_retries=1) capture = _LogCapture(logging.ERROR) metadata = {} await _drive_cyclic_fallback(router, capture, metadata=metadata, **request_kwargs) - breadcrumbs = metadata["previous_models"] - assert breadcrumbs, "no retry breadcrumbs were recorded" - dumped = json.dumps(breadcrumbs, default=str) - assert container_key in dumped, "the credential-bearing kwarg never reached the breadcrumb, so this test cannot see the leak" + records = metadata["previous_models"] + assert records, "no retry records were recorded" + dumped = json.dumps(records) + assert container_key not in dumped assert _BREADCRUMB_CREDENTIAL_CANARY not in dumped @@ -10743,7 +10751,7 @@ async def _fail_one_proxy_shaped_request(router, request_marker): shallow copy of the request, so body["metadata"] is the very same dict the router later stamps previous_models onto.""" metadata = {"request_marker": request_marker} - with pytest.raises(litellm.InternalServerError): + with pytest.raises((litellm.InternalServerError, litellm.APIConnectionError)): await router.acompletion( model="broken-group", messages=[{"role": "user", "content": "hi"}], @@ -10769,34 +10777,52 @@ def _nested_breadcrumb_lists(node): @pytest.mark.asyncio -async def test_retry_breadcrumbs_stay_per_request_and_flat_across_failing_requests(): - """Every failed attempt appends a breadcrumb to metadata["previous_models"], and the proxy's +async def test_retry_records_stay_per_request_and_flat_across_failing_requests(): + """Every failed attempt appends a record to metadata["previous_models"], and the proxy's request snapshot aliases that same metadata dict. Kept on the Router and copied wholesale, - each breadcrumb embedded every earlier one from every earlier request, so the breadcrumb + each breadcrumb once embedded every earlier one from every earlier request, so the breadcrumb tree, and with it the debug repr of the kwargs, roughly doubled on each failed attempt until a single-worker proxy spent minutes in the redaction regex and stopped answering.""" router = _always_failing_router(num_retries=2) - breadcrumbs_per_request = [ + records_per_request = [ await _fail_one_proxy_shaped_request(router, f"request-{request_number}") for request_number in range(1, 7) ] - for request_number, breadcrumbs in enumerate(breadcrumbs_per_request, start=1): - assert len(breadcrumbs) == 3, "one initial attempt plus two retries failed, each leaving one breadcrumb" - assert {breadcrumb["metadata"]["request_marker"] for breadcrumb in breadcrumbs} == {f"request-{request_number}"} - for breadcrumb in breadcrumbs: - assert _nested_breadcrumb_lists(breadcrumb) == [] - assert len({len(repr(breadcrumbs)) for breadcrumbs in breadcrumbs_per_request}) == 1 + for records in records_per_request: + assert [record["attempted_retries"] for record in records] == [0, 1, 2] + for record in records: + assert set(record) == _FLAT_ATTEMPT_RECORD_KEYS + assert _nested_breadcrumb_lists(record) == [] + assert len({len(repr(records)) for records in records_per_request}) == 1 @pytest.mark.asyncio -async def test_retry_breadcrumbs_keep_only_the_last_four_attempts(): +async def test_retry_records_keep_only_the_last_four_attempts(): router = _always_failing_router(num_retries=6) - breadcrumbs = await _fail_one_proxy_shaped_request(router, "request-1") + records = await _fail_one_proxy_shaped_request(router, "request-1") - assert len(breadcrumbs) == 4 - assert [breadcrumb["metadata"]["attempted_retries"] for breadcrumb in breadcrumbs] == [3, 4, 5, 6] + assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6] + + +@pytest.mark.asyncio +async def test_num_retries_per_request_stops_retries_at_caps_above_four(monkeypatch): + """The cap used to be read off len(previous_models), which never exceeds four, so any cap above + four was inert. Reading the Router's attempted_retries counter instead lets a cap of five refuse + retries five and six before they reach the deployment.""" + monkeypatch.setattr(litellm, "num_retries_per_request", 5) + router = _always_failing_router(num_retries=6) + + records = await _fail_one_proxy_shaped_request(router, "request-1") + + assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6] + assert ["Max retries per request hit!" in record["exception_string"] for record in records] == [ + False, + False, + True, + True, + ] @pytest.mark.asyncio diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 835e87aff88..523feb2e54a 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4061,6 +4061,53 @@ class TestMetadataNoneHandling: assert metadata == {} +_RETRY_CAP_CASES: Final = ( + pytest.param(5, {"attempted_retries": 5}, True, id="cap-above-four-reached"), + pytest.param(5, {"attempted_retries": 4}, False, id="cap-above-four-not-reached"), + pytest.param(0, {"attempted_retries": 0}, False, id="first-attempt-passes-cap-of-zero"), + pytest.param(0, {"attempted_retries": 1}, True, id="cap-of-zero-refuses-first-retry"), + pytest.param(5, {"previous_models": ("a", "b", "c", "d", "e")}, False, id="breadcrumb-count-is-not-the-cap"), + pytest.param(5, None, False, id="metadata-none"), +) + + +def _capped_completion_kwargs(metadata_key: str, metadata: object) -> dict[str, object]: + return { + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "api_key": "sk-fake", + "mock_response": "ok", + metadata_key: metadata, + } + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("cap, metadata, refused", _RETRY_CAP_CASES) +def test_num_retries_per_request_reads_attempted_retries_sync(monkeypatch, metadata_key, cap, metadata, refused): + """num_retries_per_request is enforced from the Router's attempted_retries counter in whichever + metadata bucket the call carries, so callers on litellm_metadata and caps above four both work""" + monkeypatch.setattr(litellm, "num_retries_per_request", cap) + kwargs: Final = _capped_completion_kwargs(metadata_key, metadata) + if refused: + with pytest.raises(Exception, match="Max retries per request hit!"): + litellm.completion(**kwargs) + else: + assert litellm.completion(**kwargs).choices[0].message.content == "ok" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("cap, metadata, refused", _RETRY_CAP_CASES) +async def test_num_retries_per_request_reads_attempted_retries_async(monkeypatch, metadata_key, cap, metadata, refused): + monkeypatch.setattr(litellm, "num_retries_per_request", cap) + kwargs: Final = _capped_completion_kwargs(metadata_key, metadata) + if refused: + with pytest.raises(Exception, match="Max retries per request hit!"): + await litellm.acompletion(**kwargs) + else: + assert (await litellm.acompletion(**kwargs)).choices[0].message.content == "ok" + + class TestValidateAndFixThinkingParam: """Tests for validate_and_fix_thinking_param.""" diff --git a/tests/test_litellm_rust/ocr/test_lifecycle.py b/tests/test_litellm_rust/ocr/test_lifecycle.py index e7ebc5b3018..77d9ef167d0 100644 --- a/tests/test_litellm_rust/ocr/test_lifecycle.py +++ b/tests/test_litellm_rust/ocr/test_lifecycle.py @@ -806,7 +806,7 @@ async def test_shared_call_limits_still_reject_before_reading_ocr_file( monkeypatch.setattr(litellm, "_current_cost", 2) monkeypatch.setattr(litellm, "num_retries_per_request", 1 if limit == "retries" else None) expected: Final = litellm.BudgetExceededError if limit == "budget" else RuntimeError - arguments: Final = {"document": {"type": "file", "file": File()}, "metadata": {"previous_models": ["earlier"]}} + arguments: Final = {"document": {"type": "file", "file": File()}, "metadata": {"attempted_retries": 1}} with pytest.raises(expected, match=r"Budget has been exceeded|Max retries per request hit"): await call_aocr(ocr_server, **arguments) if asynchronous else call_ocr(ocr_server, **arguments) assert reads == [] From 566da87771b65665d9d8489898a86db693b5ca06 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 01:20:46 +0000 Subject: [PATCH 021/114] test(router): expect the exact error per retry-cap case and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 3 --- tests/test_litellm/test_router.py | 9 +++------ tests/test_litellm/test_utils.py | 2 -- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index ecd9cdac88b..6e76bf9d49e 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -304,9 +304,6 @@ def get_metadata_variable_name_from_kwargs( def max_retries_per_request_hit(kwargs: Mapping[str, object], num_retries_per_request: int | None) -> bool: - """ - Whether the Router retry about to run (``attempted_retries`` >= 1 in the metadata bucket) is past the cap - """ if num_retries_per_request is None: return False metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs)) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 6150ce287ed..eb29f717a20 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -10746,12 +10746,12 @@ def _always_failing_router(num_retries): ) -async def _fail_one_proxy_shaped_request(router, request_marker): +async def _fail_one_proxy_shaped_request(router, request_marker, expected_error=litellm.InternalServerError): """The proxy hands the router a metadata dict and a proxy_server_request whose body is a shallow copy of the request, so body["metadata"] is the very same dict the router later stamps previous_models onto.""" metadata = {"request_marker": request_marker} - with pytest.raises((litellm.InternalServerError, litellm.APIConnectionError)): + with pytest.raises(expected_error): await router.acompletion( model="broken-group", messages=[{"role": "user", "content": "hi"}], @@ -10808,13 +10808,10 @@ async def test_retry_records_keep_only_the_last_four_attempts(): @pytest.mark.asyncio async def test_num_retries_per_request_stops_retries_at_caps_above_four(monkeypatch): - """The cap used to be read off len(previous_models), which never exceeds four, so any cap above - four was inert. Reading the Router's attempted_retries counter instead lets a cap of five refuse - retries five and six before they reach the deployment.""" monkeypatch.setattr(litellm, "num_retries_per_request", 5) router = _always_failing_router(num_retries=6) - records = await _fail_one_proxy_shaped_request(router, "request-1") + records = await _fail_one_proxy_shaped_request(router, "request-1", expected_error=litellm.APIConnectionError) assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6] assert ["Max retries per request hit!" in record["exception_string"] for record in records] == [ diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 523feb2e54a..3f3b8ce5343 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4084,8 +4084,6 @@ def _capped_completion_kwargs(metadata_key: str, metadata: object) -> dict[str, @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) @pytest.mark.parametrize("cap, metadata, refused", _RETRY_CAP_CASES) def test_num_retries_per_request_reads_attempted_retries_sync(monkeypatch, metadata_key, cap, metadata, refused): - """num_retries_per_request is enforced from the Router's attempted_retries counter in whichever - metadata bucket the call carries, so callers on litellm_metadata and caps above four both work""" monkeypatch.setattr(litellm, "num_retries_per_request", cap) kwargs: Final = _capped_completion_kwargs(metadata_key, metadata) if refused: From 4c022a3089cbfbc377cc700db7ade6309f3b1e1f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 12 Sep 2026 23:34:12 -0700 Subject: [PATCH 022/114] feat(pricing): add azure gpt-chat-latest global and data zone rates --- ...odel_prices_and_context_window_backup.json | 111 ++++++++++++++++++ model_prices_and_context_window.json | 111 ++++++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 31 +++++ .../test_litellm/test_model_prices_schema.py | 8 ++ 4 files changed, 261 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2220d0e1fe5..86605d77aab 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7409,6 +7409,80 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure/chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/us/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, @@ -7675,6 +7749,43 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/us/gpt-chat-latest": { + "cache_read_input_token_cost": 5.5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/eu/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2220d0e1fe5..86605d77aab 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7409,6 +7409,80 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/gpt-chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "azure/chat-latest": { + "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/us/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, @@ -7675,6 +7749,43 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/us/gpt-chat-latest": { + "cache_read_input_token_cost": 5.5e-07, + "deprecation_date": "2026-12-02", + "input_cost_per_token": 5.5e-06, + "litellm_provider": "azure", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 3.3e-05, + "reasoning_effort_levels": [ + "medium" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "azure/eu/gpt-5.6": { "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index cbe6fe198c9..3c6a13a7f01 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2101,6 +2101,37 @@ def test_generic_cost_per_token_azure_gpt_6_astra_foundry_price_sheet( assert completion_cost == pytest.approx(zone_multiplier * output_multiplier * completion_tokens * 5e-5) +@pytest.mark.parametrize( + "model,input_rate,cache_read_rate,output_rate", + [ + ("azure/gpt-chat-latest", 5e-6, 5e-7, 3e-5), + ("azure/chat-latest", 5e-6, 5e-7, 3e-5), + ("azure/us/gpt-chat-latest", 5.5e-6, 5.5e-7, 3.3e-5), + ], +) +def test_generic_cost_per_token_azure_gpt_chat_latest_price_sheet( + _local_model_cost_map, model, input_rate, cache_read_rate, output_rate +): + """The Azure OpenAI price sheet lists GPT-Chat Latest at $5 input, $0.50 cached input and $30 output per 1M + tokens on Global, and $5.50, $0.55 and $33 on Data Zone. Foundry names the product gpt-chat-latest and the + OpenAI API names the same model chat-latest, so both spellings bill the Global sheet. + """ + prompt_tokens = 100000 + cached_tokens = 40000 + completion_tokens = 1000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + + prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="azure") + + assert prompt_cost == pytest.approx((prompt_tokens - cached_tokens) * input_rate + cached_tokens * cache_read_rate) + assert completion_cost == pytest.approx(completion_tokens * output_rate) + + def test_generic_cost_per_token_azure_ai_gpt_6_astra_flex_bills_the_standard_rate(_local_model_cost_map): usage = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 0b9dbd23097..e562797fbe8 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -221,6 +221,14 @@ def test_chat_latest_declares_the_one_effort_openai_accepts(prices: dict): assert resolve_supported_reasoning_efforts(prices["chat-latest"], deployment_is_mapped=True) == ("medium",) +@pytest.mark.parametrize("key", ["azure/gpt-chat-latest", "azure/chat-latest", "azure/us/gpt-chat-latest"]) +def test_azure_gpt_chat_latest_declares_the_one_effort_azure_accepts(prices: dict, key: str): + """Azure answers every reasoning_effort on a gpt-chat-latest deployment except medium with + "Unsupported value ... Supported values are: 'medium'", the same fixed level OpenAI's chat-latest + carries, so the Foundry product name and the OpenAI API name both declare that one level.""" + assert resolve_supported_reasoning_efforts(prices[key], deployment_is_mapped=True) == ("medium",) + + BEDROCK_OPENAI_GPT_MARKERS: Final = ("openai.gpt-5.4", "openai.gpt-5.5", "openai.gpt-5.6", "openai.gpt-6-astra") BEDROCK_PROVIDERS: Final = frozenset(("bedrock", "bedrock_converse", "bedrock_mantle")) BEDROCK_ROW_PREFIXES: Final = ("bedrock_mantle/", "us.", "global.") From 76ae35dfcdefbb6b09cc576909c69b54bae5999d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 13 Sep 2026 01:14:35 -0700 Subject: [PATCH 023/114] fix(types): break the CachedTokensDetails import cycle CodeQL flagged two module-level cyclic imports introduced by defining CachedTokensDetails in litellm.types.llms.openai and importing it from litellm.types.utils and litellm.cost_calculator. The class now lives in litellm.types.llms.base, which imports nothing from litellm, and every user imports it from there. Also pins that combining realtime usages where only one response.done carries cached_tokens_details keeps the earlier modality split in both orders, and commits the regenerated dashboard API types. --- litellm/cost_calculator.py | 2 +- .../transformation.py | 2 +- litellm/types/llms/base.py | 6 +++ litellm/types/llms/openai.py | 8 +--- litellm/types/utils.py | 2 +- tests/test_litellm/test_cost_calculator.py | 43 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 - 7 files changed, 53 insertions(+), 12 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ce5c84907a1..f5319776213 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -97,6 +97,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( HttpxBinaryResponseContent, ImageGenerationRequestQuality, @@ -109,7 +110,6 @@ from litellm.types.llms.openai import ( ) from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( - CachedTokensDetails, CallTypesLiteral, LiteLLMRealtimeStreamLoggingObject, LlmProviders, diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 7a215cdbb12..64324c6cad8 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -43,9 +43,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.responses.litellm_completion_transformation.session_handler import ( ResponsesSessionHandler, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( AllMessageValues, - CachedTokensDetails, ChatCompletionAssistantMessage, ChatCompletionImageObject, ChatCompletionImageUrlObject, diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index f09727ad92b..938aa8064c9 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -75,3 +75,9 @@ class HiddenParams(OpenAIObject): data: Final = super().model_dump(**kwargs) data["_response_ms"] = self._response_ms return data + + +class CachedTokensDetails(BaseModel): + text_tokens: int | None = None + audio_tokens: int | None = None + image_tokens: int | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 7a86c27efae..9bdad700d4b 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -82,7 +82,7 @@ from typing_extensions import ( override, ) -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, CachedTokensDetails from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, @@ -1285,12 +1285,6 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): model_config = {"extra": "allow"} -class CachedTokensDetails(BaseModel): - text_tokens: int | None = None - audio_tokens: int | None = None - image_tokens: int | None = None - - class InputTokensDetails(BaseLiteLLMOpenAIResponseObject): audio_tokens: int | None = None cached_tokens: int = 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 07e6fc5838b..1d73542c9bb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -48,6 +48,7 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.types.llms.base import ( BaseLiteLLMOpenAIResponseObject, + CachedTokensDetails, LiteLLMPydanticObjectBase, ) from litellm.types.mcp import MCPServerCostInfo @@ -60,7 +61,6 @@ from .llms.base import HiddenParams from .llms.openai import ( AllMessageValues, Batch, - CachedTokensDetails, ChatCompletionAnnotation, ChatCompletionReasoningItem, ChatCompletionRedactedThinkingBlock, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index a16f1b8fe4a..68e9b6143a0 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -19,6 +19,7 @@ from litellm.cost_calculator import ( ) from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import OpenAIRealtimeStreamList from litellm.types.rerank import RerankResponse from litellm.types.utils import ( @@ -4896,6 +4897,48 @@ def test_realtime_combine_sums_nested_cached_tokens_details(): assert combined.prompt_tokens_details.cached_tokens_details.image_tokens is None +@pytest.mark.parametrize("details_first", [True, False]) +def test_realtime_combine_keeps_cached_split_when_only_one_usage_has_details(details_first: bool): + with_details: Final = { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 283, + "output_tokens": 0, + "total_tokens": 283, + "input_token_details": { + "text_tokens": 116, + "audio_tokens": 167, + "cached_tokens": 192, + "cached_tokens_details": {"text_tokens": 64, "audio_tokens": 128}, + }, + } + }, + } + without_details: Final = { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 150, + "output_tokens": 0, + "total_tokens": 150, + "input_token_details": {"text_tokens": 50, "audio_tokens": 100, "cached_tokens": 100}, + } + }, + } + results: OpenAIRealtimeStreamList = ( + [with_details, without_details] if details_first else [without_details, with_details] + ) + + combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( + results=results, + ) + + assert combined.prompt_tokens_details is not None + assert combined.prompt_tokens_details.cached_tokens == 292 + assert combined.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=64, audio_tokens=128) + + def test_usage_without_cached_tokens_details_omits_key(): usage = Usage( prompt_tokens=10, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..839aa52fa84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16781,7 +16781,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16886,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) From 6f7882db34b4d15a0dec402b6218c59afc58c012 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 13 Sep 2026 01:54:12 -0700 Subject: [PATCH 024/114] fix(types): import CachedTokensDetails on its own line in openai.py CodeQL resolves `from openai import Omit` in litellm/types/llms/openai.py to the module itself, so every importer of a name whose definition line is in the diff is reported as an unsafe cyclic import. 76ae35dfcd edited the line that defines BaseLiteLLMOpenAIResponseObject there and got two alerts at files this PR does not touch. That line is now byte-identical to main and CachedTokensDetails arrives through a relative import isort keeps separate. --- litellm/types/llms/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 9bdad700d4b..e3eac9b9205 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -82,7 +82,7 @@ from typing_extensions import ( override, ) -from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, CachedTokensDetails +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from litellm.types.responses.main import ( CustomToolCallOutputItem, GenericResponseOutputItem, @@ -91,6 +91,8 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, ) +from .base import CachedTokensDetails + FileContent = IO[bytes] | bytes | PathLike FileTypes = ( From 76b26e41abfe6e72dd846e7272dcb45069b98b73 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:07:53 +0000 Subject: [PATCH 025/114] fix(router): cool down team deployments on 429 when a sibling serves the same public model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 10 ++++ litellm/router_utils/cooldown_handlers.py | 5 +- .../router_utils/test_cooldown_handlers.py | 50 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..4929b17f7fc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1553,6 +1553,16 @@ class Router: return False return sum(len(self.model_name_to_deployment_indices.get(member) or ()) for member in group.models) > 1 + def team_model_has_alternatives(self, deployment_id: str) -> bool: + deployment: Final = self.get_deployment(model_id=deployment_id) + if deployment is None: + return False + team_id: Final = deployment.model_info.team_id + public_model_name: Final = deployment.model_info.team_public_model_name + if team_id is None or public_model_name is None: + return False + return len(self.team_model_to_deployment_indices.get((team_id, public_model_name)) or ()) > 1 + _OVERRIDABLE_ROUTING_STRATEGIES: frozenset[str] = frozenset({"simple-shuffle", *_DEFAULT_SELECTOR_ATTR_BY_STRATEGY}) def _get_request_routing_strategy_override(self, request_kwargs: dict | None) -> str | None: diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index f722b6fd20c..027f0a9ca05 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -343,8 +343,9 @@ def _should_cooldown_deployment( model_group: Final = litellm_router_instance.get_model_group(id=deployment) is_single_deployment_model_group = False if model_group is not None and len(model_group) == 1: - is_single_deployment_model_group = not litellm_router_instance.routing_group_has_alternatives( - requested_model_group + is_single_deployment_model_group = not ( + litellm_router_instance.routing_group_has_alternatives(requested_model_group) + or litellm_router_instance.team_model_has_alternatives(deployment) ) ## CHECK DEPLOYMENT-LEVEL POLICY FIRST (overrides router-level) diff --git a/tests/test_litellm/router_utils/test_cooldown_handlers.py b/tests/test_litellm/router_utils/test_cooldown_handlers.py index 7ee0ed3701b..6f66bb863cc 100644 --- a/tests/test_litellm/router_utils/test_cooldown_handlers.py +++ b/tests/test_litellm/router_utils/test_cooldown_handlers.py @@ -437,3 +437,53 @@ class TestRoutingGroupCooldownAlternatives: ) is False ) + + +class TestTeamModelCooldownAlternatives: + def _router(self, team_deployments: int): + from litellm import Router + + return Router( + model_list=[ + { + "model_name": f"model_name_team-1_{i}", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": { + "id": f"team-deploy-{i}", + "team_id": "team-1", + "team_public_model_name": "team-gpt-4o-mini", + }, + } + for i in range(team_deployments) + ] + ) + + def test_429_on_team_deployment_with_sibling_cools_down(self): + from litellm.router_utils.cooldown_handlers import _should_cooldown_deployment + + router = self._router(team_deployments=2) + assert ( + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="team-deploy-0", + exception_status=429, + original_exception=Exception("rate limited"), + requested_model_group="team-gpt-4o-mini", + ) + is True + ) + + def test_429_on_only_team_deployment_keeps_single_deployment_exemption(self): + from litellm.router_utils.cooldown_handlers import _should_cooldown_deployment + + router = self._router(team_deployments=1) + assert ( + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="team-deploy-0", + exception_status=429, + original_exception=Exception("rate limited"), + requested_model_group="team-gpt-4o-mini", + ) + is False + ) From 330ba7cbf91d4db59f3e1b433aa937fcdaddbb2d Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:21:09 +0000 Subject: [PATCH 026/114] fix(ui): show the team alias on the model info page and in its raw JSON Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../src/components/ModelInfoEditForm.tsx | 10 ++- .../src/components/model_info_view.test.tsx | 78 +++++++++++++++++++ .../src/components/model_info_view.tsx | 11 ++- 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx index d56e65237eb..fb4d90b5ea2 100644 --- a/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx +++ b/ui/litellm-dashboard/src/components/ModelInfoEditForm.tsx @@ -271,6 +271,7 @@ const displayCost = (localModelData: any, field: TouchedPricingField): string => interface ModelInfoEditFormProps { localModelData: any; modelData: { model_info: { team_id?: string | null } & Record }; + teamAlias: string | null; accessToken: string | null; isEditing: boolean; isSaving: boolean; @@ -341,6 +342,7 @@ const ChipList: React.FC<{ values: unknown; emptyLabel: string }> = ({ values, e const ModelInfoEditForm: React.FC = ({ localModelData, modelData, + teamAlias, accessToken, isEditing, isSaving, @@ -799,8 +801,12 @@ const ModelInfoEditForm: React.FC = ({
- Team ID - {modelData.model_info.team_id || "Not Set"} + Team + + {teamAlias + ? `${teamAlias} (${modelData.model_info.team_id})` + : modelData.model_info.team_id || "Not Set"} +
diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index 3db9418dfb9..f714b8e5c4a 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -42,6 +42,11 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ useModelCostMap: (...args: any[]) => mockUseModelCostMap(...args), })); +const mockUseTeams = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useTeams: () => mockUseTeams(), +})); + const mockUsePtuCostAttributionEnabled = vi.fn(); vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => ({ usePtuCostAttributionEnabled: () => mockUsePtuCostAttributionEnabled(), @@ -102,6 +107,7 @@ describe("ModelInfoView", () => { }); vi.clearAllMocks(); mockUsePtuCostAttributionEnabled.mockReturnValue(false); + mockUseTeams.mockReturnValue({ data: undefined, isLoading: false, error: null }); mockUseModelsInfo.mockReturnValue({ data: { @@ -1305,6 +1311,78 @@ describe("ModelInfoView", () => { }); }); + describe("team alias", () => { + const teamModel = { + ...defaultModelData, + model_info: { ...defaultModelData.model_info, team_id: "team-1" }, + }; + + beforeEach(() => { + mockUseModelsInfo.mockReturnValue({ data: { data: [teamModel] }, isLoading: false, error: null }); + mockModelInfoV1Call.mockResolvedValue({ data: [teamModel] }); + }); + + const readRawJson = async (user: ReturnType) => { + await user.click(await screen.findByRole("tab", { name: /raw json/i })); + const pre = await screen.findByText(/"model_name": "GPT-4"/, { selector: "pre" }); + return JSON.parse(pre.textContent ?? ""); + }; + + it("shows the team alias next to the team id and adds team_alias to the raw JSON", async () => { + mockUseTeams.mockReturnValue({ + data: [ + { team_id: "team-0", team_alias: "other" }, + { team_id: "team-1", team_alias: "alpha" }, + ], + isLoading: false, + error: null, + }); + const user = userEvent.setup(); + render(, { wrapper }); + + expect(await screen.findByText("alpha (team-1)")).toBeInTheDocument(); + + const raw = await readRawJson(user); + expect(raw.model_info).toMatchObject({ team_id: "team-1", team_alias: "alpha" }); + const keys = Object.keys(raw.model_info); + expect(keys.indexOf("team_alias")).toBe(keys.indexOf("team_id") + 1); + }); + + it("falls back to the bare team id when the team is not in the caller's team list", async () => { + mockUseTeams.mockReturnValue({ + data: [{ team_id: "team-0", team_alias: "other" }], + isLoading: false, + error: null, + }); + const user = userEvent.setup(); + render(, { wrapper }); + + expect(await screen.findByText("team-1")).toBeInTheDocument(); + + const raw = await readRawJson(user); + expect(raw.model_info.team_id).toBe("team-1"); + expect(raw.model_info).not.toHaveProperty("team_alias"); + }); + + it("shows Not Set and no team_alias for a model without a team", async () => { + mockUseModelsInfo.mockReturnValue({ data: { data: [defaultModelData] }, isLoading: false, error: null }); + mockModelInfoV1Call.mockResolvedValue({ data: [defaultModelData] }); + mockUseTeams.mockReturnValue({ + data: [{ team_id: "team-1", team_alias: "alpha" }], + isLoading: false, + error: null, + }); + const user = userEvent.setup(); + render(, { wrapper }); + + expect(await screen.findByText("Team")).toBeInTheDocument(); + expect(screen.queryByText(/alpha/)).not.toBeInTheDocument(); + + const raw = await readRawJson(user); + expect(raw.model_info).not.toHaveProperty("team_alias"); + }); + }); + it("renders the provider card logo from the bundled provider map", async () => { render(, { wrapper }); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 35afcdb2985..f25416327c0 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -169,6 +169,12 @@ export default function ModelInfoView({ // Keep modelData variable name for backwards compatibility const modelData = transformedModelData; + const teamAlias = teams?.find((team) => team.team_id === modelData?.model_info?.team_id)?.team_alias || null; + const rawModelInfoEntries = Object.entries(modelData?.model_info ?? {}).flatMap((entry) => + entry[0] === "team_id" && teamAlias ? [entry, ["team_alias", teamAlias]] : [entry], + ); + const rawModelData = modelData && { ...modelData, model_info: Object.fromEntries(rawModelInfoEntries) }; + const canEditModel = canModifyModel({ userRole, userID, isViewOnly }, teams ?? null, { teamId: modelData?.model_info?.team_id, isDbModel: modelData?.model_info?.db_model === true, @@ -765,6 +771,7 @@ export default function ModelInfoView({ - 
{JSON.stringify(modelData, null, 2)}
+ 
+                {JSON.stringify(rawModelData, null, 2)}
+
From d0a846c8be5ec1ae9036254eeb04575f6b406921 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:27:33 +0000 Subject: [PATCH 027/114] test(router): cover team_model_has_alternatives directly in the mapped router test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_router.py | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b8a0d70f5bc..7af60e01f7f 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -896,6 +896,40 @@ def test_arouter_test_team_model(): assert result is not None +def test_team_model_has_alternatives(): + def team_deployment(deployment_id: str, team_id: str, public_model_name: str): + return { + "model_name": f"model_name_{team_id}_{deployment_id}", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": { + "id": deployment_id, + "team_id": team_id, + "team_public_model_name": public_model_name, + }, + } + + router = litellm.Router( + model_list=[ + team_deployment("team-a-1", "team-a", "shared-model"), + team_deployment("team-a-2", "team-a", "shared-model"), + team_deployment("team-a-solo", "team-a", "solo-model"), + team_deployment("team-b-1", "team-b", "shared-model"), + { + "model_name": "plain-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": {"id": "plain-1"}, + }, + ], + ) + + assert router.team_model_has_alternatives("team-a-1") is True + assert router.team_model_has_alternatives("team-a-2") is True + assert router.team_model_has_alternatives("team-a-solo") is False + assert router.team_model_has_alternatives("team-b-1") is False + assert router.team_model_has_alternatives("plain-1") is False + assert router.team_model_has_alternatives("missing-deployment") is False + + def test_arouter_ignore_invalid_deployments(): """ Test that router.ignore_invalid_deployments is set to True From 10f411e60dd7a771a2b3199e3306d197a3127bea Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:34:16 +0000 Subject: [PATCH 028/114] fix(router): name the all-deployments-in-cooldown error on 429 responses RouterRateLimitError now carries the model group's deployment ids so it can tell when every deployment is cooled down, and exposes that as type=all_deployments_in_cooldown with an explicit message. A partial cooldown keeps type=rate_limit_error. Either way the proxy no longer reports type=internal_server_error next to code 429 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 4 ++ litellm/router_utils/handle_error.py | 1 + litellm/types/router.py | 21 ++++++++- .../proxy/test_common_request_processing.py | 33 +++++++++++++ tests/test_litellm/test_router.py | 46 +++++++++++++++++++ 5 files changed, 104 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 8865543badd..0983c9689c3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13878,6 +13878,7 @@ class Router: cooldown_time=_cooldown_time, enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, + model_ids=model_ids, ) if strategy == "simple-shuffle": @@ -13910,6 +13911,7 @@ class Router: cooldown_time=_cooldown_time, enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, + model_ids=model_ids, ) self._override_selector_pre_call_check(strategy, strategy_selector, deployment) verbose_router_logger.info( @@ -14024,6 +14026,7 @@ class Router: cooldown_time=_cooldown_time, enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, + model_ids=model_ids, ) # 6. Apply load balancing strategy @@ -14057,6 +14060,7 @@ class Router: cooldown_time=_cooldown_time, enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, + model_ids=model_ids, ) self._override_selector_pre_call_check(strategy, strategy_selector, deployment) diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 0e7490d31b1..bfe02675162 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -93,4 +93,5 @@ async def async_raise_no_deployment_exception( cooldown_time=_cooldown_time, enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks, cooldown_list=cooldown_list_ids, + model_ids=model_ids, ) diff --git a/litellm/types/router.py b/litellm/types/router.py index c7363502017..ddc13e18567 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -645,6 +645,7 @@ class RouterErrors(enum.Enum): user_defined_ratelimit_error = "Deployment over user-defined ratelimit." no_deployments_available = "No deployments available for selected model" + all_deployments_in_cooldown = "All deployments for selected model are in cooldown" no_deployments_with_tag_routing = "Not allowed to access model due to tags configuration" no_deployments_with_provider_budget_routing = "No deployments available - crossed budget" no_healthy_deployments = "There are no healthy deployments for this model" @@ -868,6 +869,11 @@ class RouterRateLimitErrorBasic(ValueError): super().__init__(_message) +class RouterErrorTypes(str, enum.Enum): + rate_limit_error = "rate_limit_error" + all_deployments_in_cooldown = "all_deployments_in_cooldown" + + class RouterRateLimitError(ValueError): def __init__( self, @@ -875,12 +881,25 @@ class RouterRateLimitError(ValueError): cooldown_time: float, enable_pre_call_checks: bool, cooldown_list: list, + model_ids: Sequence[str] = (), ) -> None: self.model = model self.cooldown_time = cooldown_time self.enable_pre_call_checks = enable_pre_call_checks self.cooldown_list = cooldown_list - _message = f"{RouterErrors.no_deployments_available.value}, Try again in {cooldown_time} seconds. Passed model={model}. pre-call-checks={enable_pre_call_checks}, cooldown_list={cooldown_list}" + self.all_deployments_in_cooldown = bool(model_ids) and frozenset(model_ids) <= frozenset(cooldown_list) + self.type = ( + RouterErrorTypes.all_deployments_in_cooldown.value + if self.all_deployments_in_cooldown + else RouterErrorTypes.rate_limit_error.value + ) + _reason: Final = ( + f" {RouterErrors.all_deployments_in_cooldown.value}." if self.all_deployments_in_cooldown else "" + ) + _message: Final = ( + f"{RouterErrors.no_deployments_available.value}, Try again in {cooldown_time} seconds.{_reason} " + f"Passed model={model}. pre-call-checks={enable_pre_call_checks}, cooldown_list={cooldown_list}" + ) super().__init__(_message) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 69e89d1c604..92b80685a1f 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3879,6 +3879,39 @@ class TestHandleLLMApiExceptionRetryAfter: assert proxy_exc.headers["retry-after"] == "43" assert proxy_exc.headers["x-custom"] == "1" + async def test_handle_llm_api_exception_names_cooldown_when_every_deployment_is_cooled_down(self): + from litellm.types.router import RouterRateLimitError + + exc = RouterRateLimitError( + model="gpt-4", + cooldown_time=120, + enable_pre_call_checks=False, + cooldown_list=["dep-a", "dep-b"], + model_ids=["dep-a", "dep-b"], + ) + proxy_exc = await self._invoke(exc) + body = proxy_exc.to_dict() + assert body["type"] == "all_deployments_in_cooldown" + assert body["code"] == "429" + assert "All deployments for selected model are in cooldown" in body["message"] + assert proxy_exc.headers["retry-after"] == "120" + + async def test_handle_llm_api_exception_keeps_rate_limit_type_when_cooldown_is_partial(self): + from litellm.types.router import RouterRateLimitError + + exc = RouterRateLimitError( + model="gpt-4", + cooldown_time=120, + enable_pre_call_checks=False, + cooldown_list=["dep-a"], + model_ids=["dep-a", "dep-b"], + ) + proxy_exc = await self._invoke(exc) + body = proxy_exc.to_dict() + assert body["type"] == "rate_limit_error" + assert body["code"] == "429" + assert "All deployments for selected model are in cooldown" not in body["message"] + class TestHandleLLMApiExceptionFramingHeaders: """HTTP-framing headers on the provider exception must be stripped before the diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b8a0d70f5bc..cc571ad3d3c 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7735,6 +7735,52 @@ def test_get_available_deployment_raises_when_addressed_dict_is_blocked(): router.get_available_deployment(model="dep-0", request_kwargs={}) +def _cool_down(router: Router, *deployment_ids: str) -> None: + for deployment_id in deployment_ids: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=deployment_id, + original_exception=litellm.RateLimitError(message="upstream 429", llm_provider="openai", model="gpt-4o"), + exception_status=429, + cooldown_time=60, + ) + + +async def _select_deployment(router: Router, use_async: bool) -> None: + if use_async: + await router.async_get_available_deployment(model="gpt-4o", request_kwargs={}) + return + router.get_available_deployment(model="gpt-4o", request_kwargs={}) + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_get_available_deployment_names_cooldown_when_every_deployment_is_cooled_down(use_async: bool): + from litellm.types.router import RouterErrors, RouterRateLimitError + + router: Final = _router_with_two_deployments([False, False]) + _cool_down(router, "dep-0", "dep-1") + with pytest.raises(RouterRateLimitError) as exc_info: + await _select_deployment(router, use_async) + assert exc_info.value.all_deployments_in_cooldown is True + assert exc_info.value.type == "all_deployments_in_cooldown" + assert RouterErrors.all_deployments_in_cooldown.value in str(exc_info.value) + assert str(exc_info.value).startswith("No deployments available for selected model, Try again in ") + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_get_available_deployment_keeps_generic_error_when_cooldown_is_partial(use_async: bool): + from litellm.types.router import RouterErrors, RouterRateLimitError + + router: Final = _router_with_two_deployments([False, True]) + _cool_down(router, "dep-0") + with pytest.raises(RouterRateLimitError) as exc_info: + await _select_deployment(router, use_async) + assert exc_info.value.all_deployments_in_cooldown is False + assert exc_info.value.type == "rate_limit_error" + assert RouterErrors.all_deployments_in_cooldown.value not in str(exc_info.value) + + def _router_with_two_pass_through_deployments(blocked_flags): import litellm From 9080f0904ad6bee6c5f10debdccdd1b064e1efe3 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:35:54 +0000 Subject: [PATCH 029/114] fix(router): ignore blocked siblings when checking team model cooldown alternatives Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 4 +++- .../router_utils/test_cooldown_handlers.py | 18 +++++++++++++++++- tests/test_litellm/test_router.py | 6 +++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 4929b17f7fc..63d5f86d46a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1561,7 +1561,9 @@ class Router: public_model_name: Final = deployment.model_info.team_public_model_name if team_id is None or public_model_name is None: return False - return len(self.team_model_to_deployment_indices.get((team_id, public_model_name)) or ()) > 1 + sibling_indices: Final = self.team_model_to_deployment_indices.get((team_id, public_model_name)) or () + routable_siblings: Final = self._filter_blocked_deployments([self.model_list[idx] for idx in sibling_indices]) + return len(routable_siblings) > 1 _OVERRIDABLE_ROUTING_STRATEGIES: frozenset[str] = frozenset({"simple-shuffle", *_DEFAULT_SELECTOR_ATTR_BY_STRATEGY}) diff --git a/tests/test_litellm/router_utils/test_cooldown_handlers.py b/tests/test_litellm/router_utils/test_cooldown_handlers.py index 6f66bb863cc..fdbfd618dab 100644 --- a/tests/test_litellm/router_utils/test_cooldown_handlers.py +++ b/tests/test_litellm/router_utils/test_cooldown_handlers.py @@ -440,7 +440,7 @@ class TestRoutingGroupCooldownAlternatives: class TestTeamModelCooldownAlternatives: - def _router(self, team_deployments: int): + def _router(self, team_deployments: int, blocked_ids: frozenset[str] = frozenset()): from litellm import Router return Router( @@ -452,6 +452,7 @@ class TestTeamModelCooldownAlternatives: "id": f"team-deploy-{i}", "team_id": "team-1", "team_public_model_name": "team-gpt-4o-mini", + "blocked": f"team-deploy-{i}" in blocked_ids, }, } for i in range(team_deployments) @@ -487,3 +488,18 @@ class TestTeamModelCooldownAlternatives: ) is False ) + + def test_429_with_only_a_blocked_sibling_keeps_single_deployment_exemption(self): + from litellm.router_utils.cooldown_handlers import _should_cooldown_deployment + + router = self._router(team_deployments=2, blocked_ids=frozenset({"team-deploy-1"})) + assert ( + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="team-deploy-0", + exception_status=429, + original_exception=Exception("rate limited"), + requested_model_group="team-gpt-4o-mini", + ) + is False + ) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 7af60e01f7f..7786bfb4788 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -897,7 +897,7 @@ def test_arouter_test_team_model(): def test_team_model_has_alternatives(): - def team_deployment(deployment_id: str, team_id: str, public_model_name: str): + def team_deployment(deployment_id: str, team_id: str, public_model_name: str, blocked: bool = False): return { "model_name": f"model_name_{team_id}_{deployment_id}", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, @@ -905,6 +905,7 @@ def test_team_model_has_alternatives(): "id": deployment_id, "team_id": team_id, "team_public_model_name": public_model_name, + "blocked": blocked, }, } @@ -914,6 +915,8 @@ def test_team_model_has_alternatives(): team_deployment("team-a-2", "team-a", "shared-model"), team_deployment("team-a-solo", "team-a", "solo-model"), team_deployment("team-b-1", "team-b", "shared-model"), + team_deployment("team-c-1", "team-c", "paused-sibling-model"), + team_deployment("team-c-paused", "team-c", "paused-sibling-model", blocked=True), { "model_name": "plain-model", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, @@ -926,6 +929,7 @@ def test_team_model_has_alternatives(): assert router.team_model_has_alternatives("team-a-2") is True assert router.team_model_has_alternatives("team-a-solo") is False assert router.team_model_has_alternatives("team-b-1") is False + assert router.team_model_has_alternatives("team-c-1") is False assert router.team_model_has_alternatives("plain-1") is False assert router.team_model_has_alternatives("missing-deployment") is False From e61b6bfd5ff58c61381e32aae075b2516fd3c76d Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:48:35 +0000 Subject: [PATCH 030/114] fix(router): classify pass-through cooldown against pass-through deployments only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 7 ++++++- tests/test_litellm/test_router.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 0983c9689c3..e74b2079fd4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13989,6 +13989,11 @@ class Router: model=model, llm_provider="", ) + pass_through_model_ids: Final = tuple( + deployment["model_info"]["id"] + for deployment in pass_through_deployments + if "id" in deployment.get("model_info", {}) + ) # 4. Apply health-check and cooldown filtering parent_otel_span: Final[Span | None] = _get_parent_otel_span_from_kwargs(request_kwargs) @@ -14026,7 +14031,7 @@ class Router: cooldown_time=_cooldown_time, enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, - model_ids=model_ids, + model_ids=pass_through_model_ids, ) # 6. Apply load balancing strategy diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cc571ad3d3c..35d0b8104bd 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7818,6 +7818,24 @@ def test_get_available_deployment_for_pass_through_raises_when_dict_blocked(): ) +def test_get_available_deployment_for_pass_through_names_cooldown_despite_healthy_non_pass_through(): + from litellm.types.router import RouterRateLimitError + + router: Final = _router_with_two_pass_through_deployments([False, False]) + router.add_deployment( + Deployment( + model_name="gpt-4o", + litellm_params=LiteLLM_Params(model="openai/gpt-4o-plain", api_key="sk-fake-for-tests"), + model_info=ModelInfo(id="plain-0"), + ) + ) + _cool_down(router, "pt-0", "pt-1") + with pytest.raises(RouterRateLimitError) as exc_info: + router.get_available_deployment_for_pass_through(model="gpt-4o", request_kwargs={}) + assert exc_info.value.all_deployments_in_cooldown is True + assert exc_info.value.type == "all_deployments_in_cooldown" + + def test_initialize_deployment_for_pass_through_keeps_bedrock_iam_deployment(): """ Bedrock deployments using IAM/OIDC auth have no api_key; pass-through From e41b3bd13fa6415de7b4076dda82f673fac8b957 Mon Sep 17 00:00:00 2001 From: yassin Date: Sun, 13 Sep 2026 09:50:12 +0000 Subject: [PATCH 031/114] test(router): annotate return types of team cooldown test helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/router_utils/test_cooldown_handlers.py | 6 ++---- tests/test_litellm/test_router.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/router_utils/test_cooldown_handlers.py b/tests/test_litellm/router_utils/test_cooldown_handlers.py index fdbfd618dab..6fed4be5909 100644 --- a/tests/test_litellm/router_utils/test_cooldown_handlers.py +++ b/tests/test_litellm/router_utils/test_cooldown_handlers.py @@ -440,10 +440,8 @@ class TestRoutingGroupCooldownAlternatives: class TestTeamModelCooldownAlternatives: - def _router(self, team_deployments: int, blocked_ids: frozenset[str] = frozenset()): - from litellm import Router - - return Router( + def _router(self, team_deployments: int, blocked_ids: frozenset[str] = frozenset()) -> litellm.Router: + return litellm.Router( model_list=[ { "model_name": f"model_name_team-1_{i}", diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 7786bfb4788..c6b3b7fb8d6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -897,7 +897,9 @@ def test_arouter_test_team_model(): def test_team_model_has_alternatives(): - def team_deployment(deployment_id: str, team_id: str, public_model_name: str, blocked: bool = False): + def team_deployment( + deployment_id: str, team_id: str, public_model_name: str, blocked: bool = False + ) -> DeploymentTypedDict: return { "model_name": f"model_name_{team_id}_{deployment_id}", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, From db79226b6b8786b40d10e8736595a0fe6bf07f47 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 13 Sep 2026 09:59:39 +0000 Subject: [PATCH 032/114] test(auth): freeze the cache clock in auth prefetch tests The org cache entries written by prefetch_auth_objects carry the 5s DEFAULT_IN_MEMORY_TTL. The first @log_db_metrics getter lazily imports litellm.proxy.proxy_server, which on a cold CI runner can take longer than 5s, so the org entry expired before get_org_object read it and the getter fell through to the MagicMock database. Inject a frozen clock into InMemoryCache so the test asserts the join, not import latency. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy_behavior/auth/test_auth_object_prefetch.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/proxy_behavior/auth/test_auth_object_prefetch.py b/tests/proxy_behavior/auth/test_auth_object_prefetch.py index e2d947f4284..59e9585a296 100644 --- a/tests/proxy_behavior/auth/test_auth_object_prefetch.py +++ b/tests/proxy_behavior/auth/test_auth_object_prefetch.py @@ -22,6 +22,11 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache pytestmark = pytest.mark.asyncio(loop_scope="session") +def _frozen_cache() -> UserApiKeyCache: + """The org entries carry a 5s TTL; a frozen clock keeps a slow first call from expiring them mid-test.""" + return UserApiKeyCache(in_memory_cache=InMemoryCache(clock=lambda: 1_000_000.0), redis_cache=None) + + def _dead_db() -> MagicMock: prisma = MagicMock(name="prisma_client") prisma.db.query_first = AsyncMock(return_value=None) @@ -58,7 +63,7 @@ async def test_join_binds_the_membership_to_the_requested_team(prisma): data={"user_id": user_id, "team_id": team_b, "litellm_budget_table": {"connect": {"budget_id": f"b-{run}"}}} ) - cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None) + cache = _frozen_cache() refs = AuthObjectRefs(user_id=user_id, team_id=team_a, membership_user_id=user_id, organization_id=org_id) await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma) @@ -100,7 +105,7 @@ async def test_join_reads_team_model_aliases_from_the_mapped_column(prisma): where={"team_id": team_id}, include={"litellm_model_table": True} ) - cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None) + cache = _frozen_cache() refs = AuthObjectRefs(user_id=None, team_id=team_id, membership_user_id=None, organization_id=None) await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma) @@ -144,7 +149,7 @@ async def test_join_reads_null_nested_lists_the_way_prisma_does(prisma): where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}}, include={"litellm_budget_table": True} ) - cache = UserApiKeyCache(in_memory_cache=InMemoryCache(), redis_cache=None) + cache = _frozen_cache() refs = AuthObjectRefs(user_id=user_id, team_id=team_id, membership_user_id=user_id, organization_id=None) await prefetch_auth_objects(refs=refs, user_api_key_cache=cache, prisma_client=prisma) From 0201ca60e7397912cf46e0a1bddf1d6befb86e0a Mon Sep 17 00:00:00 2001 From: ryan Date: Sun, 13 Sep 2026 23:09:27 +0000 Subject: [PATCH 033/114] fix(ui): move tags typed into key metadata JSON into the Tags field Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../templates/KeyEditViewControls.tsx | 41 +++++++++++++++++- .../templates/keyEditFieldNormalizers.test.ts | 42 +++++++++++++++++++ .../templates/keyEditFieldNormalizers.ts | 33 +++++++++++++++ .../key_edit_view.integration.test.tsx | 26 ++++++++++++ .../components/templates/key_edit_view.tsx | 22 ++++++---- 5 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/templates/keyEditFieldNormalizers.test.ts diff --git a/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx b/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx index f5d5562ccfe..39542882798 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyEditViewControls.tsx @@ -1,12 +1,15 @@ import React from "react"; -import { Control } from "react-hook-form"; +import { Control, UseFormReturn } from "react-hook-form"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { CircleHelp } from "lucide-react"; import { FormField } from "@/components/shared/form/FormField"; +import { toast } from "@/lib/toast"; import AgentSelector from "../agent_management/AgentSelector"; import NumericalInput from "../shared/numerical_input"; import SkillSelector from "../skills/SkillSelector"; +import { moveTagsOutOfMetadataJson } from "./keyEditFieldNormalizers"; import { AgentsAndGroups, KeyEditFormValues } from "./keyEditFormValues"; export const labelWithHint = (label: React.ReactNode, hint: string): React.ReactNode => ( @@ -85,6 +88,42 @@ export const KeyAgentAndSkillFields = ({ ); +type KeyEditForm = Pick< + UseFormReturn, + "control" | "getValues" | "setValue" +>; + +export const moveMetadataTagsToTagsField = (form: KeyEditForm): void => { + const moved = moveTagsOutOfMetadataJson(form.getValues("metadata"), form.getValues("tags")); + if (moved === null) return; + form.setValue("metadata", moved.metadata, { shouldDirty: true }); + form.setValue("tags", moved.tags, { shouldDirty: true }); + if (moved.movedTags.length > 0) { + toast.info(`Moved ${moved.movedTags.join(", ")} from metadata to the Tags field`); + } +}; + +export const KeyMetadataField = ({ form }: { form: KeyEditForm }) => ( + + {(field) => ( +