scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge 94d6ff942a into 3002994c0e

Browse source
This commit is contained in:
tin-berri 2026-08-28 03:31:53 +00:00 • committed by GitHub
commit 93f0433735
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 1032 additions and 393 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

6
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)

85
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
with_prompt_cache_breakpoint,
)
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
CacheControlInjectionPoint,
CacheControlMessageInjectionPoint,
)
@ -185,7 +187,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=applied_message_points,
messages=processed_messages,
@ -194,7 +196,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
@ -236,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return provider
@staticmethod
def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
system_blocks: Final = (
sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
)
@ -258,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
``max_blocks`` is reached. Injection points are honored in config order,
so earlier points win when slots are scarce.
"""
used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
limit_reached = False
for point in points:
@ -454,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system)
if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
system_already_has_cc: Final = isinstance(processed_system, list) and any(
@ -589,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
carry the mark either at the top level (Anthropic shape) or nested under
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
"""
if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
return True
if tools is not None:
return any(
@ -749,6 +751,64 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if points:
non_default_params["cache_control_injection_points"] = points
@staticmethod
def record_gateway_injection(
request_kwargs: Mapping[str, object],
added: int,
) -> None:
"""Name the deployment whose payload the gateway, not the client, put breakpoints on.
Spend accounting only asks whether litellm acted, so what it needs is which
deployment, not a count. Recording that is what makes the mark attempt-scoped: the
metadata bucket is one dict shared by every retry, failover and fallback of a
request, and ``litellm_call_id`` is shared with it, so anything request-scoped
written by one attempt is read by all of them and each boundary would have to
remember to strip it. The deployment is the part that actually changes when the
request moves, so a leg that injected nothing is never credited for one that did.
It also makes a zero delta (hook re-entry) and a negative one (a prompt manager
replacing the messages) harmless, since neither rewrites an earlier mark.
A pass that runs before a deployment is chosen, which is what the proxy does for
prompt templates, injects into the payload every leg goes on to send, so it marks
the request for all of them rather than for one.
Only what this pass actually placed counts. A ``tool_config`` point is placed by
the Bedrock converse transform, and only when the request carries tools, so the
presence of one here says nothing about whether a breakpoint reaches the wire;
claiming it marked three request shapes out of four that inject nothing. Missing
that Bedrock credit is the fail-closed direction, and the alternative is a
provider transform that carries spend-attribution state.
Reads whichever bucket the request actually carries rather than asking the shared
name resolver, which answers on key presence: ``litellm_params`` declares
``litellm_metadata`` as None on every request, so the resolver names a bucket that
is not there and the mark is dropped.
Never CREATES the bucket. The proxy seeds it on every request and is the marker's
only reader, so a request without one is a bare SDK call nothing would consume it
from. Creating it would also add a key to a dict call sites splat as ``**kwargs``,
and on the Responses API ``metadata`` is both this bucket's default name and an
explicit parameter, so the splat collides with the caller's own value.
"""
if added <= 0:
return
bucket: Final = next(
(
candidate
for candidate in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata"))
if isinstance(candidate, dict)
),
None,
)
if bucket is not None:
model_info: Final = request_kwargs.get("model_info")
bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = (
model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT)
if isinstance(model_info, dict)
else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
)
@staticmethod
def maybe_inject_cache_control(
messages: list[dict],
@ -798,17 +858,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
messages=messages,
system=system,
injection_points=injection_points,
openai_dialect=openai_dialect,
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
):
breakpoints_added: Final = (
AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
)
AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
if openai_dialect and breakpoints_added > 0:
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
if remaining:
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)

18
litellm/litellm_core_utils/litellm_logging.py
View file

@ -888,7 +888,10 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_management_logger: CustomLogger | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
non_default_params=non_default_params,
@ -898,6 +901,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@ -913,6 +917,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
if request_kwargs is not None:
AnthropicCacheControlHook.record_gateway_injection(
request_kwargs,
AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
)
self.messages = messages
return model, messages, non_default_params
@ -928,7 +937,10 @@ class Logging(LiteLLMLoggingBaseClass):
tools: list[dict] | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
tools=tools,
@ -939,6 +951,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@ -956,6 +969,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
if request_kwargs is not None:
AnthropicCacheControlHook.record_gateway_injection(
request_kwargs,
AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
)
self.messages = messages
return model, messages, non_default_params

13
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1543,6 +1543,7 @@ class AmazonConverseConfig(BaseConfig):
messages: list[AllMessageValues] | None = None,
headers: dict | None = None,
drop_params: bool = False,
litellm_params: Mapping[str, object] | None = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
@ -1605,6 +1606,16 @@ class AmazonConverseConfig(BaseConfig):
if point.get("location") == "tool_config":
cache_point = self._build_cache_point_block(point.get("control"), model)
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
# Spend attribution credits the gateway only for breakpoints it placed, and
# this is the one place a tool_config point becomes one. The hook that reads
# the configuration cannot record it: whether a cachePoint lands depends on
# this provider and on the request carrying tools, neither of which it sees.
if litellm_params is not None:
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
)
AnthropicCacheControlHook.record_gateway_injection(litellm_params, 1)
break
bedrock_tool_config: ToolConfigBlock | None = None
@ -1667,6 +1678,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
litellm_params=litellm_params,
)
bedrock_messages: Final = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
@ -1726,6 +1738,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
headers=headers,
drop_params=litellm_params.get("drop_params") is True,
litellm_params=litellm_params,
)
## TRANSFORMATION ##

2
litellm/main.py
View file

@ -531,6 +531,7 @@ async def acompletion(
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
#########################################################
# if the chat completion logging hook removed all tools,
@ -5246,6 +5247,7 @@ def completion(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
### LITELLM SYSTEM PROMPT ###

10
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -2689,6 +2689,11 @@
"title": "Total Flat Cost",
"type": "number"
},
"total_gateway_injected_caching_savings_spend": {
"default": 0.0,
"title": "Total Gateway Injected Caching Savings Spend",
"type": "number"
},
"total_pages": {
"default": 1,
"title": "Total Pages",
@ -3175,6 +3180,11 @@
"title": "Flat Cost",
"type": "number"
},
"gateway_injected_caching_savings_spend": {
"default": 0.0,
"title": "Gateway Injected Caching Savings Spend",
"type": "number"
},
"prompt_caching_savings_spend": {
"default": 0.0,
"title": "Prompt Caching Savings Spend",

2
litellm/proxy/_types.py
View file

@ -3581,6 +3581,7 @@ class SpendLogsMetadata(TypedDict):
cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
litellm_gateway_injected_cache: ReadOnly[str | None]
class SpendLogsPayload(TypedDict):
@ -4861,6 +4862,7 @@ class BaseDailySpendTransaction(TypedDict):
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
gateway_injected_caching_savings_spend: float # writable-ok: the rollup queue accumulates into this key in place, as it does for every sibling spend field
# Not required: rows queued by a pod running the previous release, or replayed from
# the Redis buffer across an upgrade, carry no such key. Every reader coalesces a
# missing value to zero, so requiring it here would describe a shape the aggregation

1
litellm/proxy/db/daily_spend_bulk_upsert.py
View file

@ -62,6 +62,7 @@ _SPEND_COLUMNS: Final = (
"spend",
"compression_savings_spend",
"prompt_caching_savings_spend",
"gateway_injected_caching_savings_spend",
"autorouter_savings_spend",
)

4
litellm/proxy/db/db_spend_update_writer.py
View file

@ -62,6 +62,7 @@ from litellm.proxy.spend_tracking.savings import (
compute_savings_spend,
extract_cache_creation_tokens,
extract_cache_read_tokens,
marks_gateway_injection,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
@ -315,6 +316,7 @@ class DBSpendUpdateWriter:
model=payload.get("model"),
custom_llm_provider=payload.get("custom_llm_provider"),
compression_saved_tokens=0,
gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")),
routing_decision=metadata.get("routing_decision"),
usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
model_id=payload.get("model_id"),
@ -1879,6 +1881,7 @@ class DBSpendUpdateWriter:
model=payload.get("model", None),
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
gateway_injected_cache=marks_gateway_injection(_metadata, payload.get("model_id")),
routing_decision=_metadata.get("routing_decision"),
model_id=payload.get("model_id"),
llm_router=_get_llm_router,
@ -1911,6 +1914,7 @@ class DBSpendUpdateWriter:
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching,
autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter,
)
return daily_transaction

4
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file

@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)
daily_transaction["gateway_injected_caching_savings_spend"] = (
payload.get("gateway_injected_caching_savings_spend", 0) or 0
) + daily_transaction.get("gateway_injected_caching_savings_spend", 0)
daily_transaction["autorouter_savings_spend"] = (
payload.get("autorouter_savings_spend", 0) or 0
) + daily_transaction.get("autorouter_savings_spend", 0)

3
litellm/proxy/litellm_pre_call_utils.py
View file

@ -51,6 +51,7 @@ from litellm.proxy.common_utils.callback_utils import (
strip_callback_config,
)
from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers
from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
# Cache special headers as a frozenset for O(1) lookup performance
_SPECIAL_HEADERS_CACHE: Final = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values())
@ -221,6 +222,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
"policy_sources",
"guardrail_scan_ids",
"routing_decision",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
"pillar_response_headers",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
@ -275,6 +277,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
"policy_sources",
"guardrail_scan_ids",
"routing_decision",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,

14
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -94,6 +94,9 @@ class DailySpendRecord(Protocol):
@property
def prompt_caching_savings_spend(self) -> float: ...
@property
def gateway_injected_caching_savings_spend(self) -> float: ...
@property
def autorouter_savings_spend(self) -> float: ...
@ -137,6 +140,7 @@ class _GroupingSetsRow(SimpleNamespace):
compression_saved_tokens: int | None
compression_savings_spend: float | None
prompt_caching_savings_spend: float | None
gateway_injected_caching_savings_spend: float | None
autorouter_savings_spend: float | None
api_requests: int | None
successful_requests: int | None
@ -189,6 +193,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
existing_metrics.gateway_injected_caching_savings_spend += ( # rebind-ok: this accumulator mutates its target in place for every metric on the row
record.gateway_injected_caching_savings_spend or 0
)
existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
@ -721,6 +728,7 @@ def _build_aggregated_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
@ -799,6 +807,7 @@ def _build_entity_rollup_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
@ -934,6 +943,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
compression_saved_tokens=record.compression_saved_tokens or 0,
compression_savings_spend=record.compression_savings_spend or 0,
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
gateway_injected_caching_savings_spend=record.gateway_injected_caching_savings_spend or 0,
autorouter_savings_spend=record.autorouter_savings_spend or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
@ -1200,6 +1210,7 @@ async def get_daily_activity(
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
total_gateway_injected_caching_savings_spend=metadata_metrics.gateway_injected_caching_savings_spend,
total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
@ -1372,6 +1383,9 @@ async def get_daily_activity_aggregated(
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
total_gateway_injected_caching_savings_spend=aggregated[
"totals"
].gateway_injected_caching_savings_spend,
total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend,
page=1,
total_pages=1,

6
litellm/proxy/schema.prisma
View file

@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)

48
litellm/proxy/spend_tracking/savings.py
View file

@ -15,6 +15,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token
from litellm.types.integrations.anthropic_cache_control_hook import (
GATEWAY_INJECTED_CACHE_METADATA_KEY,
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
)
if TYPE_CHECKING:
from litellm.router import Router
@ -25,6 +29,7 @@ class SavingsSpend(NamedTuple):
compression: float
prompt_caching: float
autorouter: float = 0.0
gateway_injected_caching: float = 0.0
def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]:
@ -391,6 +396,28 @@ def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage |
return None
def marks_gateway_injection(metadata: Mapping[str, object] | None, model_id: str | None) -> bool:
"""Whether the gateway put cache breakpoints on the payload THIS row was billed for.
``AnthropicCacheControlHook.record_gateway_injection`` stamps the deployment it
injected for, and a row carries the deployment it was billed for, so the two agree
only on the leg that was actually injected. Every retry, failover and fallback of a
request shares one metadata bucket and one ``litellm_call_id``, so the deployment is
what tells those legs apart, and a marker left by a sibling reads here as no injection
without anyone having to strip it. An injection that ran before any deployment was
chosen is in the payload every leg sends, so it is marked for all of them and credits
each. Absent on requests the gateway never acted on
(client-supplied ``cache_control``, implicit provider caching) and on rows written
before the marker shipped; all of it is the fail-closed direction.
"""
if not metadata:
return False
injected_deployment: Final = metadata.get(GATEWAY_INJECTED_CACHE_METADATA_KEY)
if not isinstance(injected_deployment, str):
return False
return injected_deployment in (GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, model_id)
def extract_cache_read_tokens(usage_object: Mapping[str, object] | None) -> int:
"""Cache-read tokens from a logged usage object, whatever shape recorded them.
@ -533,6 +560,7 @@ def compute_savings_spend(
model: str | None,
custom_llm_provider: str | None,
compression_saved_tokens: int,
gateway_injected_cache: bool,
routing_decision: Mapping[str, object] | None = None,
usage_object: Mapping[str, object] | None = None,
model_id: str | None = None,
@ -565,7 +593,23 @@ def compute_savings_spend(
A request that only writes cache and gets no hits therefore reports negative savings,
which is accurate: it really did cost more than the uncached call would have. The
daily rollup increments arithmetically, so those rows offset positive ones in the
same bucket. Auto-router savings compare the
same bucket.
Caching is reported twice. ``prompt_caching`` is every net dollar caching saved,
whoever caused it, which is what a customer means by "what did caching save me".
``gateway_injected_caching`` is the subset the gateway can claim credit for, carrying
a value only when ``gateway_injected_cache`` is set, i.e. litellm itself added the
``cache_control`` breakpoints (configured injection points or the auto prompt-caching
flag). A client that sent its own breakpoints, and a provider that
caches implicitly (OpenAI, Gemini), produce the same usage shape with no gateway
action, so they count toward the total and not toward the attributed figure.
Reporting both rather than gating the one column keeps the customer-facing number
stable across the change and leaves attribution a separate question. The attributed
figure is normally the smaller of the two, being a subset of the same requests, but
not always: a request that only writes cache and never reads it has negative net
savings, and dropping such a request from the attributed figure can lift it above
the total. Auto-router savings compare the
served ``model`` against the counterfactual baseline the router recorded on
its ``routing_decision``, and are zero unless the two differ. That record
also says whether the conversation was already underway, which is what tells
@ -602,6 +646,7 @@ def compute_savings_spend(
read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost)
prompt_caching: Final = read_discount - write_premium
gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0
# The figure the logging path recorded wins, before the usage gate on purpose: a row
# whose usage no longer parses still carries the number computed when it did.
@ -623,4 +668,5 @@ def compute_savings_spend(
compression=compression,
prompt_caching=prompt_caching,
autorouter=0.0 if autorouter is None else autorouter,
gateway_injected_caching=gateway_injected_caching,
)

1
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -137,6 +137,7 @@ def _get_spend_logs_metadata(
cost_breakdown=None,
compression_savings=None,
autorouter_savings=autorouter_savings,
litellm_gateway_injected_cache=None,
litellm_call_id=litellm_call_id,
)
verbose_proxy_logger.debug(

1
litellm/proxy/utils.py
View file

@ -1442,6 +1442,7 @@ class ProxyLogging:
prompt_variables=data.pop("prompt_variables", None) or {},
prompt_label=data.pop("prompt_label", None) or {},
prompt_version=data.pop("prompt_version", None) or {},
request_kwargs=data,
)
data.update(optional_params)

2
litellm/responses/main.py
View file

@ -537,6 +537,7 @@ async def aresponses(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
input = cast(
str | ResponseInputParam,
@ -692,6 +693,7 @@ def _apply_prompt_management_to_responses_call(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
request_kwargs=kwargs,
)
input = cast(
str | ResponseInputParam,

1
litellm/router.py
View file

@ -3998,6 +3998,7 @@ class Router:
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=prompt_label,
request_kwargs=kwargs,
)
# Filter out prompt management specific parameters from data before merging

7
litellm/types/integrations/anthropic_cache_control_hook.py
View file

@ -1,9 +1,14 @@
from typing import Literal
from typing import Final, Literal
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.types.llms.openai import ChatCompletionCachedContent
GATEWAY_INJECTED_CACHE_METADATA_KEY: Final = "litellm_gateway_injected_cache"
# No deployment had been chosen when the injection happened, so it is in the payload
# every leg of the request sends. Never a real deployment id.
GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: Final = ""
class CacheControlMessageInjectionPoint(TypedDict):
"""Type for message-level injection points."""

3
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -26,6 +26,7 @@ class SpendMetrics(BaseModel):
compression_saved_tokens: int = Field(default=0)
compression_savings_spend: float = Field(default=0.0)
prompt_caching_savings_spend: float = Field(default=0.0)
gateway_injected_caching_savings_spend: float = Field(default=0.0)
autorouter_savings_spend: float = Field(default=0.0)
total_tokens: int = Field(default=0)
successful_requests: int = Field(default=0)
@ -88,6 +89,7 @@ class DailySpendMetadata(BaseModel):
total_compression_saved_tokens: int = Field(default=0)
total_compression_savings_spend: float = Field(default=0.0)
total_prompt_caching_savings_spend: float = Field(default=0.0)
total_gateway_injected_caching_savings_spend: float = Field(default=0.0)
total_autorouter_savings_spend: float = Field(default=0.0)
page: int = Field(default=1)
total_pages: int = Field(default=1)
@ -115,6 +117,7 @@ class LiteLLM_DailyUserSpend(BaseModel):
compression_saved_tokens: int = 0
compression_savings_spend: float = 0.0
prompt_caching_savings_spend: float = 0.0
gateway_injected_caching_savings_spend: float = 0.0
autorouter_savings_spend: float = 0.0
spend: float = 0.0
api_requests: int = 0

6
schema.prisma
View file

@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)

113
tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
View file

@ -2796,3 +2796,116 @@ class TestPromptCacheBreakpointCapability:
def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected):
assert model not in litellm.model_cost
assert supports_openai_prompt_cache_breakpoint(model) is expected
class TestRecordGatewayInjection:
"""The injection marker spend accounting gates prompt-caching savings on."""
KEY = "litellm_gateway_injected_cache"
DEPLOYMENT = "dep-abc"
def test_records_only_an_actual_injection(self):
"""A zero delta is hook re-entry and a negative one is a prompt manager replacing
the messages; neither is litellm adding a breakpoint."""
kwargs: dict = {"metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
AnthropicCacheControlHook.record_gateway_injection(kwargs, 0)
AnthropicCacheControlHook.record_gateway_injection(kwargs, -3)
assert kwargs["metadata"] == {}
AnthropicCacheControlHook.record_gateway_injection(kwargs, 2)
assert kwargs["metadata"][self.KEY] == self.DEPLOYMENT
def test_a_point_this_pass_did_not_place_is_not_claimed(self):
"""A tool_config point is placed by the Bedrock converse transform, and only when
the request carries tools, so its presence here says nothing about whether a
breakpoint reaches the wire. Claiming it credited litellm on request shapes that
inject nothing, and under-crediting Bedrock tool caching is the fail-closed half.
"""
kwargs: dict = {"metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
AnthropicCacheControlHook.record_gateway_injection(kwargs, 0)
assert kwargs["metadata"] == {}
@pytest.mark.parametrize("kwargs", [{}, {"metadata": None}, {"metadata": "not-a-dict"}])
def test_never_introduces_a_metadata_key(self, kwargs):
"""Stamping must not add a key to a dict the caller splats as ``**kwargs``.
``aresponses`` takes ``metadata`` as an explicit parameter and forwards the rest
of the request as ``**kwargs``, so a bucket created here arrives twice and the
call dies with "got multiple values for keyword argument 'metadata'". Only the
proxy reads this marker and it always seeds the bucket first, so a request
without one has nothing to record.
"""
before = dict(kwargs)
AnthropicCacheControlHook.record_gateway_injection(kwargs, 3)
assert kwargs == before
def test_a_later_pass_cannot_unset_an_earlier_injection(self):
kwargs: dict = {"litellm_metadata": {"user_api_key": "k"}, "model_info": {"id": self.DEPLOYMENT}}
AnthropicCacheControlHook.record_gateway_injection(kwargs, 2)
AnthropicCacheControlHook.record_gateway_injection(kwargs, 0)
assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
def test_v1_messages_auto_injection_stamps_the_marker(self, monkeypatch):
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control(
[{"role": "user", "content": "latest turn"}],
"a long system prompt",
kwargs,
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
)
assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
def test_v1_messages_stand_down_leaves_no_marker(self, monkeypatch):
"""Client-supplied cache_control means the gateway did nothing to credit."""
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
kwargs: dict = {"litellm_metadata": {}}
AnthropicCacheControlHook.maybe_inject_cache_control(
[
{
"role": "system",
"content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}],
},
{"role": "user", "content": "latest turn"},
],
None,
kwargs,
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
)
assert self.KEY not in kwargs["litellm_metadata"]
def test_v1_messages_reentry_keeps_the_marker(self, monkeypatch):
"""A second pass over already-injected messages computes a zero delta, which must
leave the first pass's mark standing rather than reading as no injection."""
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
messages = [{"role": "user", "content": "latest turn"}]
first_msgs, first_sys = AnthropicCacheControlHook.maybe_inject_cache_control(
messages, "a long system prompt", kwargs, model="claude-sonnet-4-5", custom_llm_provider="anthropic"
)
AnthropicCacheControlHook.maybe_inject_cache_control(
first_msgs, first_sys, kwargs, model="claude-sonnet-4-5", custom_llm_provider="anthropic"
)
assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
def test_configured_points_skipping_a_marked_target_record_nothing(self):
"""Configured injection stands down on client breakpoints, so no marker lands."""
kwargs: dict = {
"litellm_metadata": {},
"cache_control_injection_points": [{"location": "message", "role": "system", "index": None}],
}
AnthropicCacheControlHook.maybe_inject_cache_control(
[
{
"role": "system",
"content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}],
},
{"role": "user", "content": "hi"},
],
None,
kwargs,
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
)
assert self.KEY not in kwargs["litellm_metadata"]

482
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -273,9 +273,7 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata():
assert cost is not None, "Cost should not be None"
expected_cost = (10 * custom_input_cost) + (5 * custom_output_cost)
assert cost == pytest.approx(
expected_cost
), f"Expected {expected_cost}, got {cost}"
assert cost == pytest.approx(expected_cost), f"Expected {expected_cost}, got {cost}"
finally:
litellm.model_cost.pop(custom_model_id, None)
@ -872,13 +870,8 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch):
# Regression check: we expect a distinct DataDogLogger, not the LLM Obs logger
assert type(datadog_logger) is DataDogLogger
assert any(
isinstance(cb, DataDogLLMObsLogger)
for cb in logging_module._in_memory_loggers
)
assert any(
type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers
)
assert any(isinstance(cb, DataDogLLMObsLogger) for cb in logging_module._in_memory_loggers)
assert any(type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers)
finally:
logging_module._in_memory_loggers.clear()
@ -889,9 +882,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
# Required env vars for Logfire integration
monkeypatch.setenv("LOGFIRE_TOKEN", "test-token")
monkeypatch.setenv(
"LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev"
) # no trailing slash on purpose
monkeypatch.setenv("LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev") # no trailing slash on purpose
# Import after env vars are set (important if module-level caching exists)
from litellm.integrations.opentelemetry import OpenTelemetry # logger class
@ -910,9 +901,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
# Sanity: we got the right logger type and it is cached
assert type(logger) is OpenTelemetry
assert any(
type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers
)
assert any(type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers)
# Core regression check: base URL env var should influence the exporter endpoint.
#
@ -923,9 +912,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
or getattr(logger, "config", None)
or getattr(logger, "_otel_config", None)
)
assert (
cfg is not None
), "Expected OpenTelemetry logger to keep an otel config on the instance"
assert cfg is not None, "Expected OpenTelemetry logger to keep an otel config on the instance"
endpoint = getattr(cfg, "endpoint", None) or getattr(cfg, "otlp_endpoint", None)
assert endpoint is not None, "Expected otel config to expose the OTLP endpoint"
@ -1083,9 +1070,7 @@ async def test_logging_non_streaming_request():
# Use the filtered call for assertions
call_args = calls_with_expected_input[0]
standard_logging_object = call_args.kwargs["kwargs"][
"standard_logging_object"
]
standard_logging_object = call_args.kwargs["kwargs"]["standard_logging_object"]
assert standard_logging_object["stream"] is not True
finally:
# Restore original callbacks to ensure test isolation
@ -1103,18 +1088,14 @@ async def test_logging_non_streaming_request():
"agenerate_content_stream",
],
)
def test_success_handler_skips_sync_callbacks_for_async_requests(
logging_obj, async_flag
):
def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, async_flag):
"""Ensure sync success callbacks are skipped when async call type flags are set."""
from litellm.integrations.custom_logger import CustomLogger
class DummyLogger(CustomLogger):
pass
logging_obj.stream = (
False # simulate non-streaming request where sync callbacks would normally run
)
logging_obj.stream = False # simulate non-streaming request where sync callbacks would normally run
logging_obj.model_call_details["litellm_params"] = {async_flag: True}
logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"]
@ -1190,21 +1171,11 @@ def test_success_handler_runs_sync_callbacks_for_sync_requests(logging_obj, call
def test_is_sync_litellm_request():
assert LitellmLogging._is_sync_litellm_request({}) is True
assert LitellmLogging._is_sync_litellm_request({"acompletion": True}) is False
assert (
LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True})
is False
)
assert (
LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False
)
assert LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True}) is False
assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False
assert LitellmLogging._is_sync_litellm_request({"agenerate_content": True}) is False
assert (
LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True})
is False
)
assert (
LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True
)
assert LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True}) is False
assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True
def test_get_litellm_params_propagates_allm_passthrough_route():
@ -1251,9 +1222,7 @@ async def test_dispatch_success_handlers_invokes_callbacks_once_for_final_stream
logging_obj.model_call_details["litellm_params"] = {"acompletion": True}
with (
patch.object(
mock_callback, "async_log_success_event", new_callable=AsyncMock
) as mock_async_log,
patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
patch.object(mock_callback, "log_success_event") as mock_sync_log,
patch.object(
logging_obj,
@ -1314,9 +1283,7 @@ async def test_dispatch_success_handlers_sync_path_invokes_callback_once_for_fin
with (
patch.object(mock_callback, "log_success_event") as mock_sync_log,
patch.object(
mock_callback, "async_log_success_event", new_callable=AsyncMock
) as mock_async_log,
patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
patch.object(
logging_obj,
"_success_handler_helper_fn",
@ -1358,20 +1325,14 @@ async def test_dispatch_prefer_async_handlers_runs_legacy_callbacks(
logging_obj.model_call_details["litellm_params"] = {}
with (
patch.object(
logging_obj, "async_success_handler", new_callable=AsyncMock
) as mock_async,
patch.object(
logging_obj, "success_handler", new_callable=MagicMock
) as mock_sync,
patch.object(logging_obj, "async_success_handler", new_callable=AsyncMock) as mock_async,
patch.object(logging_obj, "success_handler", new_callable=MagicMock) as mock_sync,
patch.object(
logging_obj,
"_should_run_sync_callbacks_for_async_calls",
return_value=True,
),
patch(
"litellm.litellm_core_utils.litellm_logging.executor.submit"
) as mock_submit,
patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_success_handlers(
result=result,
@ -1405,9 +1366,7 @@ async def test_dispatch_success_handlers_invokes_async_callback_for_pass_through
try:
with (
patch.object(
mock_callback, "async_log_success_event", new_callable=AsyncMock
) as mock_async_log,
patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
patch.object(mock_callback, "log_success_event") as mock_sync_log,
):
await logging_obj.dispatch_success_handlers(result={"id": "pt-1"})
@ -1434,20 +1393,14 @@ async def test_dispatch_failure_handlers_prefer_async_does_not_submit_sync_handl
logging_obj.model_call_details["litellm_params"] = {}
with (
patch.object(
logging_obj, "async_failure_handler", new_callable=AsyncMock
) as mock_async,
patch.object(
logging_obj, "failure_handler", new_callable=MagicMock
) as mock_sync,
patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async,
patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
patch.object(
logging_obj,
"_should_run_sync_failure_callbacks_for_async_calls",
return_value=False,
),
patch(
"litellm.litellm_core_utils.litellm_logging.executor.submit"
) as mock_submit,
patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_failure_handlers(
exception,
@ -1530,12 +1483,8 @@ async def test_dispatch_failure_handlers_submits_sync_handler_for_failure_only_c
patch.object(litellm, "success_callback", []),
patch.object(litellm, "failure_callback", [_sync_failure_callback]),
patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock),
patch.object(
logging_obj, "failure_handler", new_callable=MagicMock
) as mock_sync,
patch(
"litellm.litellm_core_utils.litellm_logging.executor.submit"
) as mock_submit,
patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_failure_handlers(
exception,
@ -1562,15 +1511,9 @@ async def test_dispatch_failure_handlers_sync_sdk_shortcut_runs_sync_handler_inl
logging_obj.model_call_details["litellm_params"] = {}
with (
patch.object(
logging_obj, "async_failure_handler", new_callable=AsyncMock
) as mock_async,
patch.object(
logging_obj, "failure_handler", new_callable=MagicMock
) as mock_sync,
patch(
"litellm.litellm_core_utils.litellm_logging.executor.submit"
) as mock_submit,
patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async,
patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_failure_handlers(
exception,
@ -1617,14 +1560,10 @@ def test_success_handler_skips_guardrail_logging_hook_when_disabled(logging_obj)
event_hook=GuardrailEventHooks.logging_only,
)
guardrail.should_run_guardrail = MagicMock(return_value=False)
guardrail.logging_hook = MagicMock(
return_value=(logging_obj.model_call_details, model_response)
)
guardrail.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response))
dummy_logger = DummyLogger()
dummy_logger.logging_hook = MagicMock(
return_value=(logging_obj.model_call_details, model_response)
)
dummy_logger.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response))
with patch.object(
logging_obj,
@ -1758,11 +1697,7 @@ def test_get_request_tags_from_metadata_and_litellm_metadata():
# Test case 2: Tags in litellm_metadata only
tags = StandardLoggingPayloadSetup._get_request_tags(
litellm_params={
"litellm_metadata": {
"tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]
}
},
litellm_params={"litellm_metadata": {"tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]}},
proxy_server_request={},
)
assert "litellm-metadata-tag-1" in tags
@ -1867,15 +1802,9 @@ def test_get_request_tags_does_not_mutate_original_tags():
user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")])
user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")])
assert (
user_agent_count_1 == 2
), f"Expected 2 User-Agent tags, got {user_agent_count_1}"
assert (
user_agent_count_2 == 2
), f"Expected 2 User-Agent tags, got {user_agent_count_2}"
assert (
user_agent_count_3 == 2
), f"Expected 2 User-Agent tags, got {user_agent_count_3}"
assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}"
assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}"
assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}"
# Verify all returned lists are independent (different objects)
assert tags1 is not tags2
@ -1908,9 +1837,7 @@ def test_get_extra_header_tags():
# Test case 3: Extra headers configured but request has no headers dict
litellm.extra_spend_tag_headers = ["x-custom", "x-tenant"]
result = StandardLoggingPayloadSetup._get_extra_header_tags(
proxy_server_request={"headers": "not-a-dict"}
)
result = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": "not-a-dict"})
assert result is None
# Test case 4: Extra headers configured but none match request headers
@ -2211,9 +2138,7 @@ def test_get_masked_values():
"presidio_anonymizer_api_base": None,
"vertex_credentials": "{sensitive_api_key}",
}
masked_values = _get_masked_values(
sensitive_object, unmasked_length=4, number_of_asterisks=4
)
masked_values = _get_masked_values(sensitive_object, unmasked_length=4, number_of_asterisks=4)
assert masked_values["presidio_anonymizer_api_base"] is None
assert masked_values["vertex_credentials"] == "{s****y}"
@ -2238,9 +2163,7 @@ async def test_e2e_generate_cold_storage_object_key_successful():
patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
):
# Mock the S3 object key generation to return a predictable result
mock_get_s3_key.return_value = (
"2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
)
mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
# Call the function
result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@ -2281,16 +2204,12 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
with (
patch("litellm.cold_storage_custom_logger", "s3_v2"),
patch(
"litellm.logging_callback_manager.get_active_custom_logger_for_callback_name"
) as mock_get_logger,
patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger,
patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
):
# Setup mocks
mock_get_logger.return_value = mock_custom_logger
mock_get_s3_key.return_value = (
"storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
)
mock_get_s3_key.return_value = "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
# Call the function
result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@ -2309,9 +2228,7 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
)
# Verify the result
assert (
result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
)
assert result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
@pytest.mark.asyncio
@ -2334,16 +2251,12 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path():
with (
patch("litellm.cold_storage_custom_logger", "s3_v2"),
patch(
"litellm.logging_callback_manager.get_active_custom_logger_for_callback_name"
) as mock_get_logger,
patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger,
patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
):
# Setup mocks
mock_get_logger.return_value = mock_custom_logger
mock_get_s3_key.return_value = (
"2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
)
mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
# Call the function
result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@ -2459,9 +2372,7 @@ def test_get_usage_as_dict():
assert result == {"prompt_tokens": 20, "completion_tokens": 30}
# Test case 5: response_obj with no usage key returns empty
result = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj={"id": "resp-1", "choices": []}
)
result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj={"id": "resp-1", "choices": []})
assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
@ -2474,26 +2385,20 @@ def test_append_system_prompt_messages():
# Test case 1: system in kwargs with existing messages
kwargs = {"system": "You are a helpful assistant"}
messages = [{"role": "user", "content": "Hello"}]
result = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=messages
)
result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
assert len(result) == 2
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
assert result[1] == {"role": "user", "content": "Hello"}
# Test case 2: system in kwargs with None messages
kwargs = {"system": "You are a helpful assistant"}
result = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=None
)
result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=None)
assert len(result) == 1
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
# Test case 3: system in kwargs with empty messages list
kwargs = {"system": "You are a helpful assistant"}
result = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=[]
)
result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=[])
assert len(result) == 1
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
@ -2503,24 +2408,18 @@ def test_append_system_prompt_messages():
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello"},
]
result = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=messages
)
result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
assert len(result) == 2
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
# Test case 5: no system in kwargs returns messages unchanged
kwargs = {}
messages = [{"role": "user", "content": "Hello"}]
result = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=kwargs, messages=messages
)
result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
assert result == messages
# Test case 6: None kwargs returns messages unchanged
result = StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=None, messages=messages
)
result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=None, messages=messages)
assert result == messages
@ -2581,12 +2480,11 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu
# Verify that standard_logging_object was set
assert "standard_logging_object" in logging_obj.model_call_details, (
"standard_logging_object should be set for pass-through endpoints "
"even when complete_streaming_response is None"
"standard_logging_object should be set for pass-through endpoints even when complete_streaming_response is None"
)
assert logging_obj.model_call_details["standard_logging_object"] is not None, (
"standard_logging_object should not be None for pass-through endpoints"
)
assert (
logging_obj.model_call_details["standard_logging_object"] is not None
), "standard_logging_object should not be None for pass-through endpoints"
# Verify that async_complete_streaming_response was set to prevent re-processing
# This is consistent with the existing code pattern for regular streaming
@ -2594,15 +2492,13 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu
"async_complete_streaming_response should be set to prevent re-processing, "
"consistent with the existing code pattern"
)
assert (
logging_obj.model_call_details["async_complete_streaming_response"] is result
), "async_complete_streaming_response should be set to the result"
assert logging_obj.model_call_details["async_complete_streaming_response"] is result, (
"async_complete_streaming_response should be set to the result"
)
# Verify that response_cost is set to None (cost calculation not possible for pass-through)
# This is consistent with the error handling in the non-pass-through code path
assert (
"response_cost" in logging_obj.model_call_details
), "response_cost should be set for pass-through endpoints"
assert "response_cost" in logging_obj.model_call_details, "response_cost should be set for pass-through endpoints"
assert logging_obj.model_call_details["response_cost"] is None, (
"response_cost should be None for pass-through endpoints since "
"StandardPassThroughResponseObject doesn't have standard usage info"
@ -2661,14 +2557,10 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp
# Verify first call set the values
assert "standard_logging_object" in logging_obj.model_call_details
assert "async_complete_streaming_response" in logging_obj.model_call_details
first_standard_logging_object = logging_obj.model_call_details[
"standard_logging_object"
]
first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"]
# Second call - should return early due to async_complete_streaming_response guard
with patch.object(
logging_obj, "get_combined_callback_list", return_value=[]
) as mock_callbacks:
with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks:
await logging_obj.async_success_handler(
result=result,
start_time=start_time,
@ -2679,10 +2571,9 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp
mock_callbacks.assert_not_called()
# Verify standard_logging_object wasn't modified by second call
assert (
logging_obj.model_call_details["standard_logging_object"]
is first_standard_logging_object
), "standard_logging_object should not be modified on re-processing"
assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, (
"standard_logging_object should not be modified on re-processing"
)
@pytest.mark.asyncio
@ -2721,9 +2612,7 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_
}
# Create a pass-through response object (simulating unparseable streaming response)
result = StandardPassThroughResponseObject(
response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]'
)
result = StandardPassThroughResponseObject(response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]')
start_time = datetime.now()
end_time = datetime.now()
@ -2743,9 +2632,9 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_
"standard_logging_object should be set for streaming pass-through endpoints "
"even when the response cannot be parsed into a ModelResponse"
)
assert (
logging_obj.model_call_details["standard_logging_object"] is not None
), "standard_logging_object should not be None for streaming pass-through endpoints"
assert logging_obj.model_call_details["standard_logging_object"] is not None, (
"standard_logging_object should not be None for streaming pass-through endpoints"
)
def test_get_error_information_error_code_priority():
@ -2787,30 +2676,22 @@ def test_get_error_information_error_code_priority():
self.message = message
super().__init__(message)
both_exception = BothAttributesException(
code="400", status_code=500, message="Bad Request"
)
both_exception = BothAttributesException(code="400", status_code=500, message="Bad Request")
result = StandardLoggingPayloadSetup.get_error_information(both_exception)
assert result["error_code"] == "400" # Should prefer 'code' over 'status_code'
# Test case 4: Exception with 'code' as empty string - should fall back to 'status_code'
empty_code_exception = BothAttributesException(
code="", status_code=404, message="Not Found"
)
empty_code_exception = BothAttributesException(code="", status_code=404, message="Not Found")
result = StandardLoggingPayloadSetup.get_error_information(empty_code_exception)
assert result["error_code"] == "404" # Should fall back to status_code
# Test case 5: Exception with 'code' as "None" string - should fall back to 'status_code'
none_string_exception = BothAttributesException(
code="None", status_code=503, message="Service Unavailable"
)
none_string_exception = BothAttributesException(code="None", status_code=503, message="Service Unavailable")
result = StandardLoggingPayloadSetup.get_error_information(none_string_exception)
assert result["error_code"] == "503" # Should fall back to status_code
# Test case 6: Exception with 'code' as None - should fall back to 'status_code'
none_code_exception = BothAttributesException(
code=None, status_code=401, message="Unauthorized"
)
none_code_exception = BothAttributesException(code=None, status_code=401, message="Unauthorized")
result = StandardLoggingPayloadSetup.get_error_information(none_code_exception)
assert result["error_code"] == "401" # Should fall back to status_code
@ -2859,9 +2740,7 @@ def test_get_error_information_prefers_message_attribute_over_str():
)
result = StandardLoggingPayloadSetup.get_error_information(exc)
assert (
result["error_message"] == msg
), f"expected message from .message attribute, got {result['error_message']!r}"
assert result["error_message"] == msg, f"expected message from .message attribute, got {result['error_message']!r}"
assert result["error_code"] == "401"
assert result["error_class"] == "ProxyExceptionLike"
@ -2936,8 +2815,7 @@ def test_get_error_information_preserves_explicit_empty_message():
exc = ProxyExceptionLike(message="", code=500)
result = StandardLoggingPayloadSetup.get_error_information(exc)
assert result["error_message"] == "", (
"explicit empty .message must survive verbatim; got "
f"{result['error_message']!r}"
f"explicit empty .message must survive verbatim; got {result['error_message']!r}"
)
@ -3200,9 +3078,7 @@ def test_process_hidden_params_recalculates_cost_after_failure_handler_zero():
choices=[{"message": {"role": "assistant", "content": "ok"}}],
usage=Usage(prompt_tokens=9698, completion_tokens=30, total_tokens=9728),
)
logging_obj._process_hidden_params_and_response_cost(
result, datetime.now(), datetime.now()
)
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
cost = logging_obj.model_call_details.get("response_cost")
assert cost is not None and cost > 0
@ -3226,9 +3102,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params():
litellm_call_id="test-hidden-zero-cost",
function_id="test-hidden-zero-cost",
)
logging_obj.model_call_details["litellm_params"] = {
"model": "gemini-2.5-flash-lite"
}
logging_obj.model_call_details["litellm_params"] = {"model": "gemini-2.5-flash-lite"}
logging_obj.optional_params = {}
result = ModelResponse(
@ -3238,9 +3112,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params():
)
result._hidden_params = {"response_cost": 0.0}
logging_obj._process_hidden_params_and_response_cost(
result, datetime.now(), datetime.now()
)
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
assert logging_obj.model_call_details.get("response_cost") == 0.0
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
@ -3289,9 +3161,7 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer
)
result._hidden_params = {"response_cost": passthrough_cost}
logging_obj._process_hidden_params_and_response_cost(
result, datetime.now(), datetime.now()
)
logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
assert logging_obj.model_call_details.get("response_cost") == passthrough_cost
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
@ -3348,9 +3218,7 @@ def test_function_setup_litellm_metadata_populates_metadata():
assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash
# metadata should be a COPY, not an alias — mutating one must not affect the other
assert (
metadata is not litellm_metadata
), "litellm_params['metadata'] should be a copy, not the same object"
assert metadata is not litellm_metadata, "litellm_params['metadata'] should be a copy, not the same object"
def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
@ -3395,9 +3263,9 @@ def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
litellm_params = logging_obj.model_call_details.get("litellm_params", {})
litellm_metadata = litellm_params.get("litellm_metadata")
assert litellm_metadata is not None
assert litellm_metadata.get("standard_logging_guardrail_information") == [
guardrail_entry
], "guardrail writes after function_setup must be visible to the logging object"
assert litellm_metadata.get("standard_logging_guardrail_information") == [guardrail_entry], (
"guardrail writes after function_setup must be visible to the logging object"
)
assert litellm_metadata.get("applied_guardrails") == ["pam-ethical-request"]
merged = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)
@ -3566,9 +3434,7 @@ def test_failure_handler_skips_sync_callbacks_for_pass_through_requests(logging_
@pytest.mark.parametrize("call_type", ["completion", "acompletion"])
def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(
logging_obj, call_type
):
def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(logging_obj, call_type):
"""Ensure sync failure callbacks still fire for normal (non-pass-through) requests."""
from litellm.integrations.custom_logger import CustomLogger
@ -3729,9 +3595,7 @@ def test_standard_logging_hidden_params_backfills_response_cost_without_mutating
)
response._hidden_params = {"response_cost": None, "model_id": "mid-test"}
payload = logging_obj._build_standard_logging_payload(
response, datetime.now(), datetime.now()
)
payload = logging_obj._build_standard_logging_payload(response, datetime.now(), datetime.now())
assert payload is not None
assert payload["hidden_params"]["response_cost"] == 0.002
@ -3785,10 +3649,7 @@ def test_merge_hidden_params_from_response_into_metadata_no_op_when_empty():
_hidden_params = {}
logging_obj._merge_hidden_params_from_response_into_metadata(_NoHp())
assert (
"hidden_params"
not in logging_obj.model_call_details["litellm_params"]["metadata"]
)
assert "hidden_params" not in logging_obj.model_call_details["litellm_params"]["metadata"]
# ── StandardLoggingPayloadSetup.get_additional_headers ───────────────────────
@ -3985,9 +3846,7 @@ def test_success_handler_computes_cost_for_dict_response():
"_build_standard_logging_payload",
return_value={"response_cost": expected_cost},
),
patch(
"litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
),
patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(
logging_obj,
"_is_recognized_call_type_for_logging",
@ -4024,9 +3883,7 @@ def test_success_handler_preserves_precomputed_cost_for_dict_response():
"_build_standard_logging_payload",
return_value={"response_cost": precomputed_cost},
),
patch(
"litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
),
patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(
logging_obj,
"_is_recognized_call_type_for_logging",
@ -4065,9 +3922,7 @@ def test_success_handler_unified_helper_runs_for_typed_results():
"_build_standard_logging_payload",
return_value={"response_cost": expected_cost},
),
patch(
"litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
),
patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(
logging_obj,
"_is_recognized_call_type_for_logging",
@ -4122,9 +3977,7 @@ class TestFirstApiCallStartTimeSetOnce:
assert first == obj.model_call_details["api_call_start_time"]
# Set on the logging object only — user metadata untouched.
assert user_meta == {}
assert (
"first_api_call_start_time" not in obj.model_call_details["litellm_params"]
)
assert "first_api_call_start_time" not in obj.model_call_details["litellm_params"]
time.sleep(0.002) # ensure a distinct retry timestamp
obj.pre_call(input="hi", api_key="sk-test")
@ -4141,18 +3994,16 @@ def test_get_error_information_for_logging_payload_ignores_spoofed_disconnect_wi
baseline = StandardLoggingPayloadSetup.get_error_information(
original_exception=ValueError("provider failure"),
)
error_information, error_str = (
StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={
"error_information": {
"error_code": "499",
"error_message": "Client disconnected the request",
"error_class": "ClientDisconnected",
}
},
original_exception=ValueError("provider failure"),
error_str="provider failure",
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={
"error_information": {
"error_code": "499",
"error_message": "Client disconnected the request",
"error_class": "ClientDisconnected",
}
},
original_exception=ValueError("provider failure"),
error_str="provider failure",
)
assert error_information == baseline
assert error_str == "provider failure"
@ -4166,22 +4017,18 @@ def test_get_error_information_for_logging_payload_client_disconnect():
"error_message": "Client disconnected the request",
"error_class": "ClientDisconnected",
}
error_information, error_str = (
StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={"client_disconnected": True, "error_information": custom_error},
original_exception=None,
error_str=None,
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={"client_disconnected": True, "error_information": custom_error},
original_exception=None,
error_str=None,
)
assert error_information == custom_error
assert error_str == "Client disconnected the request"
error_information, error_str = (
StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={"client_disconnected": True},
original_exception=None,
error_str="existing error",
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={"client_disconnected": True},
original_exception=None,
error_str="existing error",
)
assert error_information["error_code"] == "499"
assert error_str == "existing error"
@ -4189,12 +4036,10 @@ def test_get_error_information_for_logging_payload_client_disconnect():
baseline = StandardLoggingPayloadSetup.get_error_information(
original_exception=None,
)
error_information, error_str = (
StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={},
original_exception=None,
error_str=None,
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata={},
original_exception=None,
error_str=None,
)
assert error_information == baseline
assert error_str is None
@ -4229,9 +4074,7 @@ def test_get_error_information_prefers_message_attribute_over_empty_str():
def __str__(self):
return ""
info = StandardLoggingPayloadSetup.get_error_information(
original_exception=_SilentExc()
)
info = StandardLoggingPayloadSetup.get_error_information(original_exception=_SilentExc())
assert info["error_message"] == "real failure detail"
assert info["error_code"] == "401"
@ -4262,9 +4105,7 @@ def _responses_api_response_with_text(text="hello world"):
type="message",
role="assistant",
status="completed",
content=[
ResponseOutputText(annotations=[], text=text, type="output_text")
],
content=[ResponseOutputText(annotations=[], text=text, type="output_text")],
)
],
usage=ResponseAPIUsage(input_tokens=11, output_tokens=7, total_tokens=18),
@ -4279,9 +4120,7 @@ def _responses_api_response_with_text(text="hello world"):
("ResponseFailedEvent", "response.failed"),
],
)
def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(
event_cls, event_type
):
def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(event_cls, event_type):
"""Regression for #28595 / #28943. When anthropic_messages routes to the OpenAI
Responses backend and stream=True, success_handler receives a terminal Responses
API event. The handler must translate it to a ModelResponse whose choices carry
@ -4320,10 +4159,7 @@ def test_handle_anthropic_messages_response_logging_passes_model_response_throug
"""Anthropic-native path already yields a ModelResponse; it must be returned unchanged."""
logging_obj = _anthropic_messages_logging_obj()
model_response = ModelResponse()
assert (
logging_obj._handle_anthropic_messages_response_logging(result=model_response)
is model_response
)
assert logging_obj._handle_anthropic_messages_response_logging(result=model_response) is model_response
def test_handle_anthropic_messages_response_logging_degrades_on_unparseable_responses_payload():
@ -4619,9 +4455,7 @@ def test_non_image_response_has_no_output_image_count(logging_obj):
def test_zero_token_video_usage_preserves_duration_seconds(logging_obj):
"""Video usage bills by duration; the payload must keep duration_seconds even with zero tokens."""
payload = _build_payload_for_media_response(
logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}}
)
payload = _build_payload_for_media_response(logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}})
assert payload is not None
assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0
@ -5973,3 +5807,93 @@ def test_failure_handler_helper_fn_builds_payload_once_per_exception():
other_exc = _raise_and_catch(_ClientError(status_code=429, message="rate limited"))
obj._failure_handler_helper_fn(exception=other_exc, traceback_exception="")
assert obj.model_call_details["standard_logging_object"] is not first_payload
@pytest.mark.asyncio
async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_obj):
"""The savings gate reads litellm_gateway_injected_cache from the request's
metadata bucket. Recording lives in the shared prompt-hook wrappers, so chat,
/v1/responses, router prompt deployments, and proxy prompt templates all mark
injected requests the same way; a hook that injects nothing leaves no marker."""
from litellm.integrations.custom_prompt_management import CustomPromptManagement
class _InjectingHook(CustomPromptManagement):
def get_chat_completion_prompt(
self,
model,
messages,
non_default_params,
prompt_id,
prompt_variables,
dynamic_callback_params,
prompt_label=None,
prompt_version=None,
prompt_spec=None,
):
marked = [{**messages[0], "cache_control": {"type": "ephemeral"}}, *messages[1:]]
return model, marked, non_default_params
async def async_get_chat_completion_prompt(
self,
model,
messages,
non_default_params,
prompt_id,
prompt_variables,
dynamic_callback_params,
litellm_logging_obj=None,
tools=None,
prompt_label=None,
prompt_version=None,
prompt_spec=None,
):
return self.get_chat_completion_prompt(
model, messages, non_default_params, prompt_id, prompt_variables, dynamic_callback_params
)
class _PassthroughHook(CustomPromptManagement):
def get_chat_completion_prompt(
self,
model,
messages,
non_default_params,
prompt_id,
prompt_variables,
dynamic_callback_params,
prompt_label=None,
prompt_version=None,
prompt_spec=None,
):
return model, messages, non_default_params
request_kwargs = {"metadata": {}, "model_info": {"id": "dep-of-this-attempt"}}
_, marked, _ = await logging_obj.async_get_chat_completion_prompt(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
non_default_params={},
prompt_variables=None,
prompt_management_logger=_InjectingHook(),
request_kwargs=request_kwargs,
)
assert request_kwargs["metadata"]["litellm_gateway_injected_cache"] == "dep-of-this-attempt"
logging_obj.get_chat_completion_prompt(
model="claude-sonnet-5",
messages=marked,
non_default_params={},
prompt_variables=None,
prompt_management_logger=_PassthroughHook(),
request_kwargs=request_kwargs,
)
assert request_kwargs["metadata"]["litellm_gateway_injected_cache"] == "dep-of-this-attempt"
untouched = {"metadata": {}}
logging_obj.get_chat_completion_prompt(
model="claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
non_default_params={},
prompt_variables=None,
prompt_management_logger=_PassthroughHook(),
request_kwargs=untouched,
)
assert "litellm_gateway_injected_cache" not in untouched["metadata"]

37
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -887,6 +887,43 @@ def test_get_supported_openai_params_bedrock_converse():
print(f"✅ Passed for model: {model}")
@pytest.mark.parametrize(
"tools, expected_marker",
[
pytest.param(
[{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}],
"dep-bedrock",
id="tools-present-so-the-cachepoint-is-placed",
),
pytest.param(None, None, id="no-tools-so-nothing-is-placed"),
],
)
def test_tool_config_cachepoint_is_credited_only_where_it_is_placed(tools, expected_marker):
"""Spend attribution credits the gateway for breakpoints it placed, and a tool_config
point becomes one here or nowhere.
The hook that reads the configuration cannot record it: whether a cachePoint lands
depends on this provider and on the request carrying tools, neither of which the hook
sees, so marking on the point's presence credited request shapes that inject nothing.
"""
bucket: dict = {"user_api_key": "sk-test"}
optional_params = {"cache_control_injection_points": [{"location": "tool_config"}]}
if tools is not None:
optional_params["tools"] = tools
data = AmazonConverseConfig()._transform_request_helper(
model="anthropic.claude-sonnet-4-5-20250929-v1:0",
system_content_blocks=[],
optional_params=optional_params,
messages=[{"role": "user", "content": "hi"}],
litellm_params={"metadata": bucket, "litellm_metadata": None, "model_info": {"id": "dep-bedrock"}},
)
placed = "cachePoint" in json.dumps(data.get("toolConfig", {}))
assert placed is (expected_marker is not None)
assert bucket.get("litellm_gateway_injected_cache") == expected_marker
def test_transform_request_helper_includes_anthropic_beta_and_tools():
"""Test _transform_request_helper includes anthropic_beta for computer tools."""
config = AmazonConverseConfig()

2
tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
View file

@ -207,6 +207,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"compression_saved_tokens": 0,
"compression_savings_spend": 0,
"prompt_caching_savings_spend": 0,
"gateway_injected_caching_savings_spend": 0,
"autorouter_savings_spend": 0,
}
@ -258,6 +259,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"compression_saved_tokens": 0,
"compression_savings_spend": 0,
"prompt_caching_savings_spend": 0,
"gateway_injected_caching_savings_spend": 0,
"autorouter_savings_spend": 0,
}

6
tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py
View file

@ -85,10 +85,10 @@ def test_one_statement_carries_every_row_in_the_batch():
assert sql.count("INSERT INTO") == 1
assert len(re.findall(r"ON CONFLICT", sql)) == 1
# 22 bound columns per row plus the inlined updated_at, so the row count is what
# 23 bound columns per row plus the inlined updated_at, so the row count is what
# separates one multi-row statement from a hundred single-row ones.
assert len(params) == 100 * 22
assert "$2200::text" in sql
assert len(params) == 100 * 23
assert "$2300::text" in sql
assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1

104
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -2331,6 +2331,7 @@ async def test_daily_transaction_carries_compression_saved_tokens():
metadata = {
"usage_object": {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 15},
"litellm_gateway_injected_cache": "dep-of-the-compression-row",
"compression_savings": {
"tokens_before": 12000,
"tokens_after": 5000,
@ -2354,6 +2355,7 @@ async def test_daily_transaction_carries_compression_saved_tokens():
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"model_id": "dep-of-the-compression-row",
"call_type": "anthropic_messages",
"prompt_tokens": 5000,
"completion_tokens": 10,
@ -2740,3 +2742,105 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_on_responses_calls(
assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}]
assert PrismaClient.spend_log_flush_requested.is_set() is expects_flush
PrismaClient.spend_log_flush_requested.clear()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"injected_deployment, attributed",
[
pytest.param("dep-of-this-row", True, id="this-deployment-injected"),
pytest.param("dep-of-a-sibling-leg", False, id="a-sibling-deployment-injected"),
pytest.param("", True, id="injected-before-a-deployment-was-chosen"),
],
)
async def test_caching_savings_are_attributed_to_the_deployment_that_was_injected(
injected_deployment, attributed
):
"""Retries, same-group failover and cross-model-group fallbacks all reuse one metadata
bucket and one litellm_call_id, so a marker written by the leg that injected is
visible to every sibling and nothing request-scoped can tell them apart.
Naming the deployment it injected for is what keeps the credit on that leg: a row
billed for a different deployment reads it as no injection, so no seam has to strip
it and a deployment that injected nothing is never credited for the one that did.
An injection that ran before any deployment was chosen, which is what the proxy does
for prompt templates, is written into the payload every leg goes on to send, so it
marks the request for all of them and each leg keeps the credit.
"""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
mock_prisma.get_request_status = MagicMock(return_value="success")
payload = {
"request_id": "req-fallback-leg",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"model_id": "dep-of-this-row",
"call_type": "anthropic_messages",
"prompt_tokens": 5000,
"completion_tokens": 10,
"spend": 0.05,
"metadata": json.dumps(
{
"usage_object": {"cache_read_input_tokens": 4242, "cache_creation_input_tokens": 1111},
"litellm_gateway_injected_cache": injected_deployment,
}
),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload=payload,
prisma_client=mock_prisma,
type="user",
)
assert transaction is not None
assert transaction["prompt_caching_savings_spend"] != 0.0
assert (transaction["gateway_injected_caching_savings_spend"] != 0.0) is attributed
@pytest.mark.asyncio
async def test_daily_transaction_attributes_caching_savings_only_with_an_injection_marker():
"""Cached usage with no litellm_gateway_injected_cache marker is still a real saving.
Client-sent cache_control and implicit provider caching leave no marker, so the row
keeps the total the customer actually got while the gateway-attributed column stays
empty, which is what separates what caching saved from what litellm can claim.
"""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
mock_prisma.get_request_status = MagicMock(return_value="success")
payload = {
"request_id": "req-ungated-caching",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"call_type": "anthropic_messages",
"prompt_tokens": 5000,
"completion_tokens": 10,
"spend": 0.05,
"metadata": json.dumps(
{"usage_object": {"cache_read_input_tokens": 4242, "cache_creation_input_tokens": 1111}}
),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload=payload,
prisma_client=mock_prisma,
type="user",
)
assert transaction is not None
assert transaction["cache_read_input_tokens"] == 4242
assert transaction["cache_creation_input_tokens"] == 1111
assert transaction["prompt_caching_savings_spend"] != 0.0
assert transaction["gateway_injected_caching_savings_spend"] == 0.0

11
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -155,6 +155,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"gateway_injected_caching_savings_spend": 0.0,
"autorouter_savings_spend": 0.0,
"failed_requests": 0,
}
@ -485,6 +486,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_1.compression_saved_tokens = 0
mock_record_1.compression_savings_spend = 0.0
mock_record_1.prompt_caching_savings_spend = 0.0
mock_record_1.gateway_injected_caching_savings_spend = 0.0
mock_record_1.autorouter_savings_spend = 0.0
mock_record_1.api_requests = 10
mock_record_1.successful_requests = 9
@ -508,6 +510,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_2.compression_saved_tokens = 0
mock_record_2.compression_savings_spend = 0.0
mock_record_2.prompt_caching_savings_spend = 0.0
mock_record_2.gateway_injected_caching_savings_spend = 0.0
mock_record_2.autorouter_savings_spend = 0.0
mock_record_2.api_requests = 5
mock_record_2.successful_requests = 5
@ -571,6 +574,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"gateway_injected_caching_savings_spend": 0.0,
"autorouter_savings_spend": 0.0,
"failed_requests": 0,
}
@ -657,6 +661,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr
compression_saved_tokens=0,
compression_savings_spend=0.0,
prompt_caching_savings_spend=0.0,
gateway_injected_caching_savings_spend=0.0,
autorouter_savings_spend=0.0,
api_requests=1,
successful_requests=1,
@ -1089,6 +1094,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
"compression_saved_tokens": None,
"compression_savings_spend": None,
"prompt_caching_savings_spend": None,
"gateway_injected_caching_savings_spend": None,
"autorouter_savings_spend": None,
"api_requests": None,
"successful_requests": None,
@ -1133,6 +1139,7 @@ def _no_spend_record():
compression_saved_tokens=None,
compression_savings_spend=None,
prompt_caching_savings_spend=None,
gateway_injected_caching_savings_spend=None,
autorouter_savings_spend=None,
api_requests=None,
successful_requests=None,
@ -1242,6 +1249,7 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost=
compression_saved_tokens=0,
compression_savings_spend=0,
prompt_caching_savings_spend=0,
gateway_injected_caching_savings_spend=0,
autorouter_savings_spend=0,
total_tokens=0,
api_requests=0,
@ -1307,6 +1315,7 @@ def _grouping_row(
compression_saved_tokens=0,
compression_savings_spend=0.0,
prompt_caching_savings_spend=0.0,
gateway_injected_caching_savings_spend=0.0,
autorouter_savings_spend=0.0,
api_requests=0,
successful_requests=0,
@ -1466,6 +1475,7 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib
compression_saved_tokens=0,
compression_savings_spend=0,
prompt_caching_savings_spend=0,
gateway_injected_caching_savings_spend=0,
autorouter_savings_spend=0,
total_tokens=0,
api_requests=0,
@ -1869,6 +1879,7 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown():
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
"gateway_injected_caching_savings_spend": 0.0,
"autorouter_savings_spend": 0.0,
"failed_requests": 0,
"prompt_tokens": 0,

125
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -8,6 +8,7 @@ from litellm.proxy.spend_tracking.savings import (
_baseline_usage,
compute_autorouter_savings,
compute_savings_spend,
marks_gateway_injection,
)
from litellm.router import Router
from litellm.types.utils import Usage
@ -59,6 +60,7 @@ def test_compression_savings_priced_at_input_rate():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=4389,
gateway_injected_cache=True,
)
assert result.compression == pytest.approx(4389 * input_cost)
assert result.compression > 0
@ -74,6 +76,7 @@ def test_prompt_caching_savings_priced_at_input_minus_cache_read():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 8200},
)
assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost))
@ -126,6 +129,7 @@ def test_prompt_caching_savings_nets_out_the_cache_write_premium():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=usage_object,
)
assert result.prompt_caching == pytest.approx(_net_caching_savings_against_biller(usage_object))
@ -141,6 +145,7 @@ def test_prompt_caching_savings_go_negative_on_a_write_only_request():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=usage_object,
)
true_savings = _net_caching_savings_against_biller(usage_object)
@ -156,6 +161,7 @@ def test_prompt_caching_savings_negative_when_writes_outweigh_reads():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=usage_object,
)
true_savings = _net_caching_savings_against_biller(usage_object)
@ -172,6 +178,7 @@ def test_read_only_request_is_unchanged_by_the_write_premium():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=20000, written=0),
)
assert result.prompt_caching == pytest.approx(20000 * (input_cost - cache_read_cost))
@ -185,12 +192,14 @@ def test_openai_style_cache_write_tokens_are_netted_out():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 5000, "cache_creation_input_tokens": 800},
)
nested_only = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={
"prompt_tokens_details": {"cached_tokens": 5000, "cache_write_tokens": 800},
},
@ -221,6 +230,7 @@ def test_model_without_a_cache_write_price_takes_no_premium():
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=5000, written=5000),
)
assert result.prompt_caching == pytest.approx(5000 * (input_cost - cache_read_cost))
@ -244,6 +254,7 @@ def test_zero_cache_write_price_is_read_as_unpublished():
model="deepseek-chat",
custom_llm_provider="deepseek",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=0, written=10000),
)
assert result.prompt_caching == pytest.approx(0.0)
@ -268,6 +279,7 @@ def test_zero_cache_read_price_stays_literal():
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=10000, written=0),
)
# free reads => the whole input rate is saved, not zero
@ -293,6 +305,7 @@ def test_sub_input_cache_write_price_is_an_extra_saving():
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=4000),
)
assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write))
@ -306,6 +319,7 @@ def test_negative_cache_write_count_clamps_to_zero():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 1000, "cache_creation_input_tokens": -5000},
)
assert result.prompt_caching == pytest.approx(1000 * (input_cost - cache_read_cost))
@ -316,6 +330,7 @@ def test_unknown_model_fails_open_to_zero():
model="totally-made-up-model-xyz",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 1000},
)
assert result.compression == 0.0
@ -327,6 +342,7 @@ def test_missing_model_fails_open_to_zero():
model=None,
custom_llm_provider=None,
compression_saved_tokens=1000,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 1000},
)
assert result.compression == 0.0
@ -338,6 +354,7 @@ def test_negative_token_counts_clamp_to_zero():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=-500,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": -500},
)
assert result.compression == 0.0
@ -516,6 +533,7 @@ def test_autorouter_savings_zero_without_baseline():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision=None,
usage_object=_cached_usage_object(),
)
@ -530,6 +548,7 @@ def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch):
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True},
usage_object=_cached_usage_object(),
)
@ -543,6 +562,7 @@ def test_the_driver_is_off_until_a_baseline_is_configured():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True},
usage_object=_cached_usage_object(),
)
@ -557,6 +577,7 @@ def test_malformed_usage_object_does_not_fail_the_spend_write():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True},
usage_object={"prompt_tokens": ["not", "a", "number"]},
)
@ -573,6 +594,7 @@ def test_model_without_cache_read_pricing_yields_no_caching_savings():
model=model,
custom_llm_provider="azure",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 5000},
)
assert result.prompt_caching == 0.0
@ -882,6 +904,7 @@ def test_a_baseline_recorded_on_the_decision_turns_the_driver_on():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"},
usage_object=_cached_usage_object(),
)
@ -895,6 +918,7 @@ def test_the_configured_baseline_overrides_the_recorded_one(monkeypatch):
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={
"conversation_continuing": True,
"savings_baseline_model": "anthropic/claude-opus-5",
@ -923,6 +947,7 @@ def test_a_non_string_recorded_baseline_is_ignored():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": ["anthropic/claude-opus-5"]},
usage_object=_cached_usage_object(),
)
@ -954,6 +979,7 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=20000),
model_id=deployment_id,
llm_router=lambda: router,
@ -965,6 +991,7 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=20000),
)
assert result.prompt_caching > at_public_rates.prompt_caching
@ -995,6 +1022,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision=decision,
usage_object=_cached_usage_object(),
llm_router=lambda: router,
@ -1003,6 +1031,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={k: v for k, v in decision.items() if k != "savings_baseline_deployment_id"},
usage_object=_cached_usage_object(),
llm_router=lambda: router,
@ -1021,6 +1050,7 @@ def test_recorded_savings_win_over_recomputation():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
routing_decision=_routed_decision(),
usage_object=_cached_usage_object(),
recorded_autorouter_savings=0.5,
@ -1035,6 +1065,7 @@ def test_recorded_savings_survive_an_unusable_usage_object():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
routing_decision=_routed_decision(),
usage_object={"prompt_tokens": ["not", "a", "number"]},
recorded_autorouter_savings=0.25,
@ -1047,6 +1078,7 @@ def test_a_boolean_is_not_a_recorded_savings_figure():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
routing_decision=None,
usage_object=_cached_usage_object(),
recorded_autorouter_savings=True,
@ -1063,6 +1095,7 @@ def test_rows_written_before_the_field_shipped_recompute():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
routing_decision=_routed_decision(),
usage_object=_cached_usage_object(),
)
@ -1127,3 +1160,95 @@ def test_logging_payload_never_stamps_internal_calls():
cost_breakdown=None,
)
assert internal is None
def test_caching_savings_require_a_gateway_injected_breakpoint():
"""The same cached usage is attributed to the gateway only when it added a breakpoint.
Client-sent cache_control and implicit provider caching (OpenAI, Gemini) produce
cache reads the gateway had no hand in. Those still count as caching savings the
customer really got, so the total is unchanged, but nothing about them is the
gateway's doing and the attributed figure has to stay empty.
"""
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
credited = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=8200, written=0),
)
expected = 8200 * (input_cost - cache_read_cost)
assert credited.prompt_caching == pytest.approx(expected)
assert credited.gateway_injected_caching == pytest.approx(expected)
unattributed = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
usage_object=_caching_usage(read=8200, written=0),
)
assert unattributed.prompt_caching == pytest.approx(expected)
assert unattributed.gateway_injected_caching == 0.0
def test_unattributed_write_only_request_still_reports_its_loss_in_the_total():
"""A write-only request really did cost more than not caching, whoever asked for it.
The attributed figure drops it because the gateway added no breakpoint, and dropping a
negative is why the attributed number can sit above the total rather than below it.
"""
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
usage_object=_caching_usage(read=0, written=20000),
)
assert result.prompt_caching < 0
assert result.gateway_injected_caching == 0.0
assert result.gateway_injected_caching > result.prompt_caching
def test_injected_request_keeps_its_negative_net():
"""A gateway-injected write-heavy request still reports its real loss."""
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=True,
usage_object=_caching_usage(read=0, written=20000),
)
assert result.prompt_caching < 0
def test_attribution_does_not_touch_compression_or_autorouter_legs():
input_cost, _ = _anthropic_costs("claude-sonnet-5")
result = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=4389,
gateway_injected_cache=False,
usage_object=_caching_usage(read=8200, written=0),
)
assert result.compression == pytest.approx(4389 * input_cost)
assert result.prompt_caching > 0
assert result.gateway_injected_caching == 0.0
def test_marks_gateway_injection_credits_only_the_deployment_that_was_injected():
"""Every retry, failover and fallback of a request shares one metadata bucket and one
litellm_call_id, so the deployment is what tells those legs apart. A marker naming a
sibling has to read here as no injection; that is what keeps the credit on the leg
that earned it without any seam having to strip it. Anything that is not this row's
own deployment, the missing key included, is fail-closed."""
assert marks_gateway_injection(None, "dep-a") is False
assert marks_gateway_injection({}, "dep-a") is False
assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, "dep-a") is True
assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, "dep-b") is False
assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, None) is False
# injected before a deployment was chosen, so it is in the payload every leg sends
assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, "dep-a") is True
assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, None) is True
assert marks_gateway_injection({"litellm_call_id": "c1"}, "dep-a") is False
assert marks_gateway_injection({"litellm_gateway_injected_cache": True}, "dep-a") is False

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -2865,7 +2865,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -2961,7 +2961,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -3055,7 +3055,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

41
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -3915,3 +3915,44 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully():
assert (
metadata.get("original_model_group") is None
), "original_model_group should be None when not provided"
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_injected_cache_breakpoints_survive_into_spend_log_metadata(bucket):
"""The injection marker only gates savings if it reaches the spend-log row.
_get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an
undeclared key is dropped silently. Both buckets are covered because chat routes
stamp metadata while /v1/messages routes stamp litellm_metadata, and
record_gateway_injection writes into whichever the request carries.
"""
payload = get_logging_payload(
kwargs={
"model": "claude-sonnet-5",
"litellm_params": {
bucket: {
"user_api_key": "test-key",
"litellm_gateway_injected_cache": "dep-of-this-row",
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-injected", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["litellm_gateway_injected_cache"] == "dep-of-this-row"
def test_passthrough_caching_carries_no_injection_marker():
"""The negative class the gate depends on: a request whose cache_control the client
supplied must read as unmarked, not merely unlabelled by accident."""
payload = get_logging_payload(
kwargs={
"model": "claude-sonnet-5",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-passthrough", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["litellm_gateway_injected_cache"] is None

4
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -954,6 +954,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies": ["spoofed-policy"],
"policy_sources": {"spoofed-policy": "request"},
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
"litellm_gateway_injected_cache": "forged-deployment-id",
"_session_deployment_affinity_ttl": 999999,
"internal_call_origin": "autorouter_classifier",
"_guardrail_pipelines": [{"name": "spoofed"}],
@ -968,6 +969,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"disable_global_guardrails": True,
"enable_prompt_caching": True,
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
"litellm_gateway_injected_cache": "forged-deployment-id",
"metadata": copy.deepcopy(malicious_metadata),
"litellm_metadata": copy.deepcopy(malicious_metadata),
}
@ -986,6 +988,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
assert "disable_global_guardrails" not in updated
assert "enable_prompt_caching" not in updated
assert "routing_decision" not in updated
assert "litellm_gateway_injected_cache" not in updated
stripped_keys = {
"disable_global_guardrails",
@ -1000,6 +1003,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies",
"policy_sources",
"routing_decision",
"litellm_gateway_injected_cache",
"_session_deployment_affinity_ttl",
"internal_call_origin",
"_guardrail_pipelines",

2
ui/litellm-dashboard/eslint-budgets.json
View file

@ -3,6 +3,6 @@
"no-console": { "max": 12, "target": 0 },
"complexity": { "max": 140, "target": 80 },
"max-depth": { "max": 70, "target": 30 },
"local/no-large-inline-object-arg": { "max": 560, "target": 300 },
"local/no-large-inline-object-arg": { "max": 559, "target": 300 },
"local/no-long-condition-chain": { "max": 265, "target": 120 }
}

49
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx
View file

@ -137,28 +137,33 @@ describe("UsageTab", () => {
});
it("sums compression and caching dollars across days into the summary cards", () => {
const { getByText } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
compression_saved_tokens: 40000,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
compression_saved_tokens: 100000,
}),
]);
// Total caching and the LiteLLM-injected share deliberately differ so these
// assertions pin which one each figure uses: the caching headline and the
// Total-saved tile take the injected share, the secondary keeps the total.
const firstDay: Partial<SpendMetrics> = {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
gateway_injected_caching_savings_spend: 0.004,
compression_saved_tokens: 40000,
};
const secondDay: Partial<SpendMetrics> = {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
gateway_injected_caching_savings_spend: 0.006,
compression_saved_tokens: 100000,
};
const { getByText } = renderWith([day("2026-07-12", firstDay), day("2026-07-13", secondDay)]);
expect(getByText("$0.1560")).toBeInTheDocument();
expect(getByText("$0.1500")).toBeInTheDocument();
expect(getByText("$0.1400")).toBeInTheDocument();
expect(getByText("$0.0100")).toBeInTheDocument();
expect(getByText("$0.0160")).toBeInTheDocument();
expect(getByText("140,000 tokens compressed")).toBeInTheDocument();
});
const twoDays = () => [
day("2026-07-12", { compression_savings_spend: 0.04, prompt_caching_savings_spend: 0.006 }),
day("2026-07-13", { compression_savings_spend: 0.1, prompt_caching_savings_spend: 0.01 }),
day("2026-07-12", { compression_savings_spend: 0.04, gateway_injected_caching_savings_spend: 0.006 }),
day("2026-07-13", { compression_savings_spend: 0.1, gateway_injected_caching_savings_spend: 0.01 }),
];
it("opens on a running total anchored at $0 at the start of the range", () => {
@ -179,7 +184,7 @@ describe("UsageTab", () => {
// synthetic start anchor gives the line a zero origin to climb from.
const oneDay = new Date(2026, 6, 24);
const { getByTestId } = renderWith(
[day("2026-07-24", { compression_savings_spend: 0.2, prompt_caching_savings_spend: 0.05 })],
[day("2026-07-24", { compression_savings_spend: 0.2, gateway_injected_caching_savings_spend: 0.05 })],
{ from: oneDay, to: oneDay },
);
@ -194,8 +199,8 @@ describe("UsageTab", () => {
// still read left to right in time, and the running total must climb toward
// the newest day, not fall away from it.
const newestFirst = [
day("2026-07-13", { prompt_caching_savings_spend: 0.1 }),
day("2026-07-12", { prompt_caching_savings_spend: 0.04 }),
day("2026-07-13", { gateway_injected_caching_savings_spend: 0.1 }),
day("2026-07-12", { gateway_injected_caching_savings_spend: 0.04 }),
];
const { getByTestId, getByRole } = renderWith(newestFirst);
@ -260,7 +265,7 @@ describe("UsageTab", () => {
const { getByRole, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.02,
gateway_injected_caching_savings_spend: 0.02,
autorouter_savings_spend: -0.05,
}),
]);
@ -312,7 +317,7 @@ describe("UsageTab", () => {
const { getByText, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.02,
gateway_injected_caching_savings_spend: 0.02,
autorouter_savings_spend: -0.05,
}),
]);
@ -329,12 +334,12 @@ describe("UsageTab", () => {
const { getByText, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
gateway_injected_caching_savings_spend: 0.006,
autorouter_savings_spend: 0.02,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
gateway_injected_caching_savings_spend: 0.01,
autorouter_savings_spend: 0.05,
}),
]);

34
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx
View file

@ -9,10 +9,7 @@ import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import useCan from "@/app/(dashboard)/hooks/useCan";
import { getToolSpend, ToolSpendResponse } from "@/components/networking";
import {
autorouterOf,
buildDailyToolSeries,
cachingOf,
compressionOf,
formatRangeLabel,
localIsoDay,
MAX_POINTS_WITH_DOTS,
@ -21,13 +18,15 @@ import {
SAVINGS_SERIES,
SavingsAccumulation,
SavingsPoint,
savingsSeriesOf,
shortDate,
sumOverDays,
toCumulative,
topToolsBySpend,
usd,
withStartAnchor,
} from "./costOptimizationUtils";
import SavingsTiles, { useSavingsTotals } from "@/components/shared/SavingsTiles";
import SavingsTiles from "@/components/shared/SavingsTiles";
import { DailyActivityRange } from "./useDailyActivityRange";
interface UsageTabProps {
@ -73,26 +72,9 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
const toolSpend = toolSpendState?.key === rangeKey ? toolSpendState.data : null;
const toolSpendLoading = toolSpendEnabled && toolSpend === null;
const totals = useSavingsTotals(results);
const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
// The daily rollup arrives newest first; sort on the raw ISO date so the axis
// reads oldest to newest and the running total accumulates forward in time
// rather than backward. Sort here, before shortDate() drops the year and makes
// the labels unsortable.
const perInterval = useMemo<SavingsPoint[]>(
() =>
[...results]
.sort((a, b) => a.date.localeCompare(b.date))
.map((d) => ({
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
"Auto-router": autorouterOf(d.metrics),
})),
[results],
);
const perInterval = useMemo<SavingsPoint[]>(() => savingsSeriesOf(results), [results]);
// Cumulative anchors on a synthetic $0 point at the range start so a short
// range (down to a single day) rises from zero instead of floating as one dot.
@ -116,14 +98,12 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
// that actually saved are plotted; the range total keeps the signed truth.
const byDriver = useMemo(
() =>
SAVINGS_DRIVERS.map(({ name, color }) => ({
SAVINGS_DRIVERS.map(({ name, color, of }) => ({
driver: name,
color,
usd: { Compression: totals.compression, "Prompt caching": totals.caching, "Auto-router": totals.autorouter }[
name
],
usd: sumOverDays(results, of),
})).filter((d) => d.usd > 0),
[totals],
[results],
);
const plottedDriverTotal = useMemo(() => byDriver.reduce((sum, d) => sum + d.usd, 0), [byDriver]);

23
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts
View file

@ -11,6 +11,7 @@ import {
formatRangeLabel,
isAnthropicModel,
localIsoDay,
savingsSeriesOf,
toCumulative,
topToolsBySpend,
usd,
@ -66,6 +67,28 @@ const modelDay = (date: string, models: Record<string, Partial<SpendMetrics>>):
},
});
describe("savingsSeriesOf", () => {
it("plots the LiteLLM-injected caching share, sorted oldest first", () => {
// Total and injected caching deliberately differ: every chart derives from
// SAVINGS_DRIVERS, so the caching series must follow the injected figure.
const sharedSavings: Partial<SpendMetrics> = {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.5,
autorouter_savings_spend: 0.05,
};
const newestFirst = [day("2026-07-02", {}), day("2026-07-01", {})].map((d, i) => ({
...d,
metrics: metrics({ ...sharedSavings, gateway_injected_caching_savings_spend: i === 0 ? 0.2 : 0.3 }),
}));
const series = savingsSeriesOf(newestFirst);
expect(series.map((p) => p.date)).toEqual(["Jul 1", "Jul 2"]);
expect(series[0]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.3, "Auto-router": 0.05 });
expect(series[1]).toMatchObject({ Compression: 0.1, "Prompt caching": 0.2, "Auto-router": 0.05 });
});
});
describe("computeCacheLeakage", () => {
it("aggregates a key's tokens and savings across multiple days", () => {
const results = [

31
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
View file

@ -17,6 +17,7 @@ export const shortDate = (iso: string): string =>
export const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
export const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
export const gatewayAttributedCachingOf = (m: SpendMetrics): number => m.gateway_injected_caching_savings_spend ?? 0;
export const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0;
export const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
@ -192,14 +193,38 @@ export type SavingsPoint = {
* mapping. Colour travels with the driver so filtering cannot separate them.
*/
export const SAVINGS_DRIVERS = [
{ name: "Compression", color: "emerald" },
{ name: "Prompt caching", color: "blue" },
{ name: "Auto-router", color: "amber" },
{ name: "Compression", color: "emerald", of: compressionOf },
{ name: "Prompt caching", color: "blue", of: gatewayAttributedCachingOf },
{ name: "Auto-router", color: "amber", of: autorouterOf },
] as const;
export const SAVINGS_SERIES = SAVINGS_DRIVERS.map((d) => d.name);
export const SAVINGS_COLORS = SAVINGS_DRIVERS.map((d) => d.color);
type SavingsDriverName = (typeof SAVINGS_DRIVERS)[number]["name"];
export const sumOverDays = (results: readonly DailyData[], of: (m: SpendMetrics) => number): number =>
results.reduce((sum, d) => sum + of(d.metrics), 0);
/**
* One point per day, each driver plotting the metric its SAVINGS_DRIVERS entry
* names. The rollup arrives newest first, so sort on the raw ISO date before
* shortDate() drops the year and makes the labels unsortable; the running total
* then accumulates forward in time. Deriving every chart's series and every
* total from the same driver list is what keeps a tile, a timeline and the
* donut from quietly plotting different metrics for the same driver name.
*/
export const savingsSeriesOf = (results: readonly DailyData[]): SavingsPoint[] =>
[...results]
.sort((a, b) => a.date.localeCompare(b.date))
.map((d) => ({
date: shortDate(d.date),
...(Object.fromEntries(SAVINGS_DRIVERS.map(({ name, of }) => [name, of(d.metrics)])) as Record<
SavingsDriverName,
number
>), // fromEntries widens keys to string; the entries are exactly the driver names
}));
/**
* Running total of each series across the selected window. The total restarts
* at the beginning of the range rather than carrying in earlier spend, which is

1
ui/litellm-dashboard/src/components/UsagePage/types.ts
View file

@ -12,6 +12,7 @@ export interface SpendMetrics {
compression_saved_tokens?: number;
compression_savings_spend?: number;
prompt_caching_savings_spend?: number;
gateway_injected_caching_savings_spend?: number;
autorouter_savings_spend?: number;
}

41
ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx
View file

@ -7,28 +7,29 @@ import {
autorouterOf,
cachingOf,
compressionOf,
gatewayAttributedCachingOf,
SAVINGS_DRIVERS,
savedTokensOf,
sumOverDays,
usd,
} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils";
import { DailyData } from "@/components/UsagePage/types";
import { formatNumberWithCommas } from "@/utils/dataUtils";
// Exported because the by-driver donut has to slice the same numbers the tiles print, and two
// totalling paths over the same rows is how a chart and the tile above it end up disagreeing.
export const useSavingsTotals = (results: DailyData[]) =>
useMemo(() => {
const sumOf = (of: (metrics: DailyData["metrics"]) => number) => results.reduce((sum, d) => sum + of(d.metrics), 0);
const compression = sumOf(compressionOf);
const caching = sumOf(cachingOf);
const autorouter = sumOf(autorouterOf);
return {
compression,
caching,
autorouter,
savedTokens: sumOf(savedTokensOf),
total: compression + caching + autorouter,
};
}, [results]);
// The total sums SAVINGS_DRIVERS, so it is by construction the sum of what the
// charts plot; the donut and timelines derive from the same list in costOptimizationUtils.
const useSavingsTotals = (results: DailyData[]) =>
useMemo(
() => ({
compression: sumOverDays(results, compressionOf),
caching: sumOverDays(results, cachingOf),
autorouter: sumOverDays(results, autorouterOf),
gatewayAttributedCaching: sumOverDays(results, gatewayAttributedCachingOf),
savedTokens: sumOverDays(results, savedTokensOf),
total: SAVINGS_DRIVERS.reduce((sum, { of }) => sum + sumOverDays(results, of), 0),
}),
[results],
);
const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading: boolean }) => {
const totals = useSavingsTotals(results);
@ -39,6 +40,7 @@ const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading:
label="Total saved"
value={usd(totals.total)}
hint={isLoading ? "Loading..." : "Compression + prompt caching + auto-router"}
info="The sum of the three tiles beside it. Its caching term is the LiteLLM-injected share, so this total is what the gateway itself delivered; caching that clients or providers brought on their own appears only in the caching tile's Total figure."
/>
<SummaryCard
label="Compression savings"
@ -48,9 +50,10 @@ const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading:
/>
<SummaryCard
label="Prompt caching savings"
value={usd(totals.caching)}
hint="Cache reads, net of write premium"
info="What caching saved against paying the input rate for every token: the discount on tokens served from cache, less the premium providers charge to write a cache entry. Can be negative on traffic that writes more cache than it reuses."
value={usd(totals.gatewayAttributedCaching)}
hint="LiteLLM injected"
secondary={{ label: "Total", value: usd(totals.caching) }}
info="What caching saved against paying the input rate for every token: the discount on tokens served from cache, less the premium providers charge to write a cache entry. The headline figure is the share LiteLLM earned by inserting the breakpoints itself, through configured injection points or auto prompt caching. The total beside it also counts requests that arrived with their own cache_control and providers that cache implicitly. Either can be negative on traffic that writes more cache than it reuses, which is why the headline is not always the smaller of the two."
/>
<SummaryCard
label="Auto-router savings"

20
ui/litellm-dashboard/src/components/shared/SummaryCard.tsx
View file

@ -12,6 +12,8 @@ export interface SummaryCardProps {
hint?: string;
/** Rendered behind an info affordance. Use it for how a figure is derived, not for restating the label. */
info?: string;
/** A related figure the headline is a share of, shown beside it. */
secondary?: { label: string; value: string };
}
/**
@ -21,7 +23,7 @@ export interface SummaryCardProps {
*/
const slugOf = (label: string): string => label.toLowerCase().replace(/\s+/g, "-");
const SummaryCard = ({ label, value, hint, info }: SummaryCardProps) => (
const SummaryCard = ({ label, value, hint, info, secondary }: SummaryCardProps) => (
<Card data-testid={`summary-card-${slugOf(label)}`}>
<CardHeader className="flex flex-row items-center justify-between space-y-0">
<CardTitle className="text-sm font-medium text-muted-foreground">{label}</CardTitle>
@ -41,8 +43,20 @@ const SummaryCard = ({ label, value, hint, info }: SummaryCardProps) => (
)}
</CardHeader>
<CardContent>
<p className="text-2xl font-semibold text-foreground">{value}</p>
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
<div className="flex items-end gap-4">
<div>
<p className="text-2xl font-semibold text-foreground">{value}</p>
{hint && <p className="mt-1 text-xs text-muted-foreground">{hint}</p>}
</div>
{secondary && (
<div className="self-stretch border-l pl-4">
<div className="flex h-full flex-col justify-end">
<p className="text-lg font-medium text-muted-foreground">{secondary.value}</p>
<p className="mt-1 text-xs text-muted-foreground">{secondary.label}</p>
</div>
</div>
)}
</div>
</CardContent>
</Card>
);

9
ui/litellm-dashboard/src/components/templates/KeySavingsTab.integration.test.tsx
View file

@ -66,6 +66,7 @@ describe("KeySavingsTab", () => {
const firstDay: Partial<SpendMetrics> = {
compression_savings_spend: 1.5,
prompt_caching_savings_spend: 0.25,
gateway_injected_caching_savings_spend: 0.1,
autorouter_savings_spend: 2,
compression_saved_tokens: 400,
cache_read_input_tokens: 300,
@ -74,6 +75,7 @@ describe("KeySavingsTab", () => {
const secondDay: Partial<SpendMetrics> = {
compression_savings_spend: 0.5,
prompt_caching_savings_spend: 0.75,
gateway_injected_caching_savings_spend: 0.3,
autorouter_savings_spend: 1,
compression_saved_tokens: 600,
cache_read_input_tokens: 200,
@ -85,10 +87,13 @@ describe("KeySavingsTab", () => {
renderTab();
expect(screen.getByTestId("summary-card-total-saved")).toHaveTextContent("$6.00");
expect(screen.getByTestId("summary-card-total-saved")).toHaveTextContent("$5.40");
expect(screen.getByTestId("summary-card-compression-savings")).toHaveTextContent("$2.00");
expect(screen.getByTestId("summary-card-compression-savings")).toHaveTextContent("1,000 tokens compressed");
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$1.00");
// the card leads with what LiteLLM's own injection earned and carries the total beneath it,
// so a key whose caching came mostly from its own cache_control does not read as gateway-earned
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$0.40");
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$1.00Total");
expect(screen.getByTestId("summary-card-auto-router-savings")).toHaveTextContent("$3.00");
});

19
ui/litellm-dashboard/src/components/templates/KeySavingsTab.tsx
View file

@ -9,9 +9,6 @@ import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle }
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { hasProxyWideSpendView, spendScopeUserId } from "@/utils/roles";
import {
autorouterOf,
cachingOf,
compressionOf,
formatRangeLabel,
localIsoDay,
MAX_POINTS_WITH_DOTS,
@ -19,6 +16,7 @@ import {
SAVINGS_SERIES,
SavingsAccumulation,
SavingsPoint,
savingsSeriesOf,
shortDate,
toCumulative,
usd,
@ -50,20 +48,7 @@ const KeySavingsTab: React.FC<KeySavingsTabProps> = ({ accessToken, keyToken, us
const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
// Sort on the raw ISO date before shortDate() drops the year: the rollup arrives newest
// first, and the running total has to accumulate forward in time.
const perInterval = useMemo<SavingsPoint[]>(
() =>
[...results]
.sort((a, b) => a.date.localeCompare(b.date))
.map((d) => ({
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
"Auto-router": autorouterOf(d.metrics),
})),
[results],
);
const perInterval = useMemo<SavingsPoint[]>(() => savingsSeriesOf(results), [results]);
const overTime = useMemo(() => {
if (accumulation !== "cumulative") return perInterval;

10
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -25818,6 +25818,11 @@ export interface components {
* @default 0
*/
total_flat_cost: number;
/**
* Total Gateway Injected Caching Savings Spend
* @default 0
*/
total_gateway_injected_caching_savings_spend: number;
/**
* Total Pages
* @default 1
@ -34915,6 +34920,11 @@ export interface components {
* @default 0
*/
flat_cost: number;
/**
* Gateway Injected Caching Savings Spend
* @default 0
*/
gateway_injected_caching_savings_spend: number;
/**
* Prompt Caching Savings Spend
* @default 0