diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql
new file mode 100644
index 00000000000..dee5abfa269
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql
@@ -0,0 +1,18 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index d9959677116..5582cf930d7 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py
index ef2edbf1007..d8452faa99c 100644
--- a/litellm/integrations/anthropic_cache_control_hook.py
+++ b/litellm/integrations/anthropic_cache_control_hook.py
@@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
with_prompt_cache_breakpoint,
)
from litellm.types.integrations.anthropic_cache_control_hook import (
+ GATEWAY_INJECTED_CACHE_METADATA_KEY,
+ GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
CacheControlInjectionPoint,
CacheControlMessageInjectionPoint,
)
@@ -185,7 +187,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
- breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
+ breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=applied_message_points,
messages=processed_messages,
@@ -194,7 +196,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
if (
openai_dialect
- and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
+ and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
@@ -236,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return provider
@staticmethod
- def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
+ def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
system_blocks: Final = (
sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
)
@@ -258,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
``max_blocks`` is reached. Injection points are honored in config order,
so earlier points win when slots are scarce.
"""
- used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
+ used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
limit_reached = False
for point in points:
@@ -454,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
- message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
- system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
+ message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
+ system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system)
if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
system_already_has_cc: Final = isinstance(processed_system, list) and any(
@@ -589,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
carry the mark either at the top level (Anthropic shape) or nested under
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
"""
- if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
+ if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
return True
if tools is not None:
return any(
@@ -749,6 +751,54 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if points:
non_default_params["cache_control_injection_points"] = points
+ @staticmethod
+ def record_gateway_injection(
+ request_kwargs: dict[str, Any], # mutable-ok: stamps the marker into the caller's live request kwargs
+ added: int,
+ ) -> None:
+ """Name the deployment whose payload the gateway, not the client, put breakpoints on.
+
+ Spend accounting only asks whether litellm acted, so what it needs is which
+ deployment, not a count. Recording that is what makes the mark attempt-scoped: the
+ metadata bucket is one dict shared by every retry, failover and fallback of a
+ request, and ``litellm_call_id`` is shared with it, so anything request-scoped
+ written by one attempt is read by all of them and each boundary would have to
+ remember to strip it. The deployment is the part that actually changes when the
+ request moves, so a leg that injected nothing is never credited for one that did.
+
+ It also makes a zero delta (hook re-entry) and a negative one (a prompt manager
+ replacing the messages) harmless, since neither rewrites an earlier mark.
+
+ A pass that runs before a deployment is chosen, which is what the proxy does for
+ prompt templates, injects into the payload every leg goes on to send, so it marks
+ the request for all of them rather than for one.
+
+ Only what this pass actually placed counts. A ``tool_config`` point is placed by
+ the Bedrock converse transform, and only when the request carries tools, so the
+ presence of one here says nothing about whether a breakpoint reaches the wire;
+ claiming it marked three request shapes out of four that inject nothing. Missing
+ that Bedrock credit is the fail-closed direction, and the alternative is a
+ provider transform that carries spend-attribution state.
+
+ Never CREATES the bucket. The proxy seeds it on every request and is the marker's
+ only reader, so a request without one is a bare SDK call nothing would consume it
+ from. Creating it would also add a key to a dict call sites splat as ``**kwargs``,
+ and on the Responses API ``metadata`` is both this bucket's default name and an
+ explicit parameter, so the splat collides with the caller's own value.
+ """
+ if added <= 0:
+ return
+ from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+
+ bucket: Final = request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs))
+ if isinstance(bucket, dict):
+ model_info: Final = request_kwargs.get("model_info")
+ bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = (
+ model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT)
+ if isinstance(model_info, dict)
+ else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
+ )
+
@staticmethod
def maybe_inject_cache_control(
messages: list[dict],
@@ -798,17 +848,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
)
- breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
+ breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
messages=messages,
system=system,
injection_points=injection_points,
openai_dialect=openai_dialect,
)
- if (
- openai_dialect
- and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
- ):
+ breakpoints_added: Final = (
+ AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
+ )
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
+ if openai_dialect and breakpoints_added > 0:
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
if remaining:
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 626af7530a4..21f0a6a53b9 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -864,7 +864,10 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_management_logger: CustomLogger | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
+ request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
+ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
non_default_params=non_default_params,
@@ -874,6 +877,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
+ breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@@ -889,6 +893,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
+ if request_kwargs is not None:
+ AnthropicCacheControlHook.record_gateway_injection(
+ request_kwargs,
+ AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
+ )
self.messages = messages
return model, messages, non_default_params
@@ -904,7 +913,10 @@ class Logging(LiteLLMLoggingBaseClass):
tools: list[dict] | None = None,
prompt_label: str | None = None,
prompt_version: int | None = None,
+ request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs
) -> tuple[str, list[AllMessageValues], dict]:
+ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
+
custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management(
model=model,
tools=tools,
@@ -915,6 +927,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if custom_logger:
+ breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
(
model,
messages,
@@ -932,6 +945,11 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_label=prompt_label,
prompt_version=prompt_version,
)
+ if request_kwargs is not None:
+ AnthropicCacheControlHook.record_gateway_injection(
+ request_kwargs,
+ AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before,
+ )
self.messages = messages
return model, messages, non_default_params
diff --git a/litellm/main.py b/litellm/main.py
index 98f92e50599..d3a5bcbcab6 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -531,6 +531,7 @@ async def acompletion(
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
+ request_kwargs=kwargs,
)
#########################################################
# if the chat completion logging hook removed all tools,
@@ -5245,6 +5246,7 @@ def completion(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
+ request_kwargs=kwargs,
)
### LITELLM SYSTEM PROMPT ###
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 61acac3ff74..f210f6349ab 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3551,6 +3551,7 @@ class SpendLogsMetadata(TypedDict):
cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
+ litellm_gateway_injected_cache: ReadOnly[str | None]
class SpendLogsPayload(TypedDict):
@@ -4831,6 +4832,7 @@ class BaseDailySpendTransaction(TypedDict):
# cost-savings metrics (dollars, priced per request before aggregation)
compression_savings_spend: float
prompt_caching_savings_spend: float
+ gateway_injected_caching_savings_spend: float # writable-ok: the rollup queue accumulates into this key in place, as it does for every sibling spend field
# Not required: rows queued by a pod running the previous release, or replayed from
# the Redis buffer across an upgrade, carry no such key. Every reader coalesces a
# missing value to zero, so requiring it here would describe a shape the aggregation
diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py
index 55d325177c6..a143643577e 100644
--- a/litellm/proxy/db/daily_spend_bulk_upsert.py
+++ b/litellm/proxy/db/daily_spend_bulk_upsert.py
@@ -62,6 +62,7 @@ _SPEND_COLUMNS: Final = (
"spend",
"compression_savings_spend",
"prompt_caching_savings_spend",
+ "gateway_injected_caching_savings_spend",
"autorouter_savings_spend",
)
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index 0c8c9a853ec..3f2777ff1f3 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -62,6 +62,7 @@ from litellm.proxy.spend_tracking.savings import (
compute_savings_spend,
extract_cache_creation_tokens,
extract_cache_read_tokens,
+ marks_gateway_injection,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
@@ -315,6 +316,7 @@ class DBSpendUpdateWriter:
model=payload.get("model"),
custom_llm_provider=payload.get("custom_llm_provider"),
compression_saved_tokens=0,
+ gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")),
routing_decision=metadata.get("routing_decision"),
usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
model_id=payload.get("model_id"),
@@ -1879,6 +1881,7 @@ class DBSpendUpdateWriter:
model=payload.get("model", None),
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
+ gateway_injected_cache=marks_gateway_injection(_metadata, payload.get("model_id")),
routing_decision=_metadata.get("routing_decision"),
model_id=payload.get("model_id"),
llm_router=_get_llm_router,
@@ -1911,6 +1914,7 @@ class DBSpendUpdateWriter:
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
+ gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching,
autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter,
)
return daily_transaction
diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
index 6a97d010b35..70a529900b2 100644
--- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
+++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
@@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("prompt_caching_savings_spend", 0) or 0
) + daily_transaction.get("prompt_caching_savings_spend", 0)
+ daily_transaction["gateway_injected_caching_savings_spend"] = (
+ payload.get("gateway_injected_caching_savings_spend", 0) or 0
+ ) + daily_transaction.get("gateway_injected_caching_savings_spend", 0)
+
daily_transaction["autorouter_savings_spend"] = (
payload.get("autorouter_savings_spend", 0) or 0
) + daily_transaction.get("autorouter_savings_spend", 0)
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 064b53e07b7..26a9bc7f1c6 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -50,6 +50,7 @@ from litellm.proxy.common_utils.callback_utils import (
strip_callback_config,
)
from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers
+from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY
# Cache special headers as a frozenset for O(1) lookup performance
_SPECIAL_HEADERS_CACHE: Final = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values())
@@ -220,6 +221,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
"policy_sources",
"guardrail_scan_ids",
"routing_decision",
+ GATEWAY_INJECTED_CACHE_METADATA_KEY,
"pillar_response_headers",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
@@ -274,6 +276,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
"policy_sources",
"guardrail_scan_ids",
"routing_decision",
+ GATEWAY_INJECTED_CACHE_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py
index d3968bf323b..91cd80b3c81 100644
--- a/litellm/proxy/management_endpoints/common_daily_activity.py
+++ b/litellm/proxy/management_endpoints/common_daily_activity.py
@@ -94,6 +94,9 @@ class DailySpendRecord(Protocol):
@property
def prompt_caching_savings_spend(self) -> float: ...
+ @property
+ def gateway_injected_caching_savings_spend(self) -> float: ...
+
@property
def autorouter_savings_spend(self) -> float: ...
@@ -137,6 +140,7 @@ class _GroupingSetsRow(SimpleNamespace):
compression_saved_tokens: int | None
compression_savings_spend: float | None
prompt_caching_savings_spend: float | None
+ gateway_injected_caching_savings_spend: float | None
autorouter_savings_spend: float | None
api_requests: int | None
successful_requests: int | None
@@ -189,6 +193,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
+ existing_metrics.gateway_injected_caching_savings_spend += ( # rebind-ok: this accumulator mutates its target in place for every metric on the row
+ record.gateway_injected_caching_savings_spend or 0
+ )
existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
@@ -721,6 +728,7 @@ def _build_aggregated_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
+ SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
@@ -799,6 +807,7 @@ def _build_entity_rollup_sql_query(
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(compression_savings_spend)::float AS compression_savings_spend,
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
+ SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend,
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
@@ -934,6 +943,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
compression_saved_tokens=record.compression_saved_tokens or 0,
compression_savings_spend=record.compression_savings_spend or 0,
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
+ gateway_injected_caching_savings_spend=record.gateway_injected_caching_savings_spend or 0,
autorouter_savings_spend=record.autorouter_savings_spend or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
@@ -1200,6 +1210,7 @@ async def get_daily_activity(
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
+ total_gateway_injected_caching_savings_spend=metadata_metrics.gateway_injected_caching_savings_spend,
total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
@@ -1372,6 +1383,9 @@ async def get_daily_activity_aggregated(
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
+ total_gateway_injected_caching_savings_spend=aggregated[
+ "totals"
+ ].gateway_injected_caching_savings_spend,
total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend,
page=1,
total_pages=1,
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index d9959677116..5582cf930d7 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py
index b0f1546e15e..7f412da32d1 100644
--- a/litellm/proxy/spend_tracking/savings.py
+++ b/litellm/proxy/spend_tracking/savings.py
@@ -15,6 +15,10 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token
+from litellm.types.integrations.anthropic_cache_control_hook import (
+ GATEWAY_INJECTED_CACHE_METADATA_KEY,
+ GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
+)
if TYPE_CHECKING:
from litellm.router import Router
@@ -25,6 +29,7 @@ class SavingsSpend(NamedTuple):
compression: float
prompt_caching: float
autorouter: float = 0.0
+ gateway_injected_caching: float = 0.0
def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]:
@@ -391,6 +396,28 @@ def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage |
return None
+def marks_gateway_injection(metadata: Mapping[str, object] | None, model_id: str | None) -> bool:
+ """Whether the gateway put cache breakpoints on the payload THIS row was billed for.
+
+ ``AnthropicCacheControlHook.record_gateway_injection`` stamps the deployment it
+ injected for, and a row carries the deployment it was billed for, so the two agree
+ only on the leg that was actually injected. Every retry, failover and fallback of a
+ request shares one metadata bucket and one ``litellm_call_id``, so the deployment is
+ what tells those legs apart, and a marker left by a sibling reads here as no injection
+ without anyone having to strip it. An injection that ran before any deployment was
+ chosen is in the payload every leg sends, so it is marked for all of them and credits
+ each. Absent on requests the gateway never acted on
+ (client-supplied ``cache_control``, implicit provider caching) and on rows written
+ before the marker shipped; all of it is the fail-closed direction.
+ """
+ if not metadata:
+ return False
+ injected_deployment: Final = metadata.get(GATEWAY_INJECTED_CACHE_METADATA_KEY)
+ if not isinstance(injected_deployment, str):
+ return False
+ return injected_deployment in (GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, model_id)
+
+
def extract_cache_read_tokens(usage_object: Mapping[str, object] | None) -> int:
"""Cache-read tokens from a logged usage object, whatever shape recorded them.
@@ -533,6 +560,7 @@ def compute_savings_spend(
model: str | None,
custom_llm_provider: str | None,
compression_saved_tokens: int,
+ gateway_injected_cache: bool,
routing_decision: Mapping[str, object] | None = None,
usage_object: Mapping[str, object] | None = None,
model_id: str | None = None,
@@ -565,7 +593,23 @@ def compute_savings_spend(
A request that only writes cache and gets no hits therefore reports negative savings,
which is accurate: it really did cost more than the uncached call would have. The
daily rollup increments arithmetically, so those rows offset positive ones in the
- same bucket. Auto-router savings compare the
+ same bucket.
+
+ Caching is reported twice. ``prompt_caching`` is every net dollar caching saved,
+ whoever caused it, which is what a customer means by "what did caching save me".
+ ``gateway_injected_caching`` is the subset the gateway can claim credit for, carrying
+ a value only when ``gateway_injected_cache`` is set, i.e. litellm itself added the
+ ``cache_control`` breakpoints (configured injection points or the auto prompt-caching
+ flag). A client that sent its own breakpoints, and a provider that
+ caches implicitly (OpenAI, Gemini), produce the same usage shape with no gateway
+ action, so they count toward the total and not toward the attributed figure.
+
+ Reporting both rather than gating the one column keeps the customer-facing number
+ stable across the change and leaves attribution a separate question. The attributed
+ figure is normally the smaller of the two, being a subset of the same requests, but
+ not always: a request that only writes cache and never reads it has negative net
+ savings, and dropping such a request from the attributed figure can lift it above
+ the total. Auto-router savings compare the
served ``model`` against the counterfactual baseline the router recorded on
its ``routing_decision``, and are zero unless the two differ. That record
also says whether the conversation was already underway, which is what tells
@@ -602,6 +646,7 @@ def compute_savings_spend(
read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost)
prompt_caching: Final = read_discount - write_premium
+ gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0
# The figure the logging path recorded wins, before the usage gate on purpose: a row
# whose usage no longer parses still carries the number computed when it did.
@@ -623,4 +668,5 @@ def compute_savings_spend(
compression=compression,
prompt_caching=prompt_caching,
autorouter=0.0 if autorouter is None else autorouter,
+ gateway_injected_caching=gateway_injected_caching,
)
diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py
index 38da38ead2b..9858999111c 100644
--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py
+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py
@@ -136,6 +136,7 @@ def _get_spend_logs_metadata(
cost_breakdown=None,
compression_savings=None,
autorouter_savings=autorouter_savings,
+ litellm_gateway_injected_cache=None,
litellm_call_id=litellm_call_id,
)
verbose_proxy_logger.debug(
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 8cbf5b685fd..78a7e9ae18f 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -1434,6 +1434,7 @@ class ProxyLogging:
prompt_variables=data.pop("prompt_variables", None) or {},
prompt_label=data.pop("prompt_label", None) or {},
prompt_version=data.pop("prompt_version", None) or {},
+ request_kwargs=data,
)
data.update(optional_params)
diff --git a/litellm/responses/main.py b/litellm/responses/main.py
index 8a682c701e0..4ec37eba060 100644
--- a/litellm/responses/main.py
+++ b/litellm/responses/main.py
@@ -541,6 +541,7 @@ async def aresponses(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
+ request_kwargs=kwargs,
)
input = cast(
str | ResponseInputParam,
@@ -664,6 +665,7 @@ def _apply_prompt_management_to_responses_call(
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
prompt_version=kwargs.get("prompt_version", None),
+ request_kwargs=kwargs,
)
input = cast(
str | ResponseInputParam,
diff --git a/litellm/router.py b/litellm/router.py
index 2dc79670c07..ab850237972 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -3959,6 +3959,7 @@ class Router:
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=prompt_label,
+ request_kwargs=kwargs,
)
# Filter out prompt management specific parameters from data before merging
diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py
index 3ab0c02f28d..ef414f22c3b 100644
--- a/litellm/types/integrations/anthropic_cache_control_hook.py
+++ b/litellm/types/integrations/anthropic_cache_control_hook.py
@@ -1,9 +1,14 @@
-from typing import Literal
+from typing import Final, Literal
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.types.llms.openai import ChatCompletionCachedContent
+GATEWAY_INJECTED_CACHE_METADATA_KEY: Final = "litellm_gateway_injected_cache"
+# No deployment had been chosen when the injection happened, so it is in the payload
+# every leg of the request sends. Never a real deployment id.
+GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: Final = ""
+
class CacheControlMessageInjectionPoint(TypedDict):
"""Type for message-level injection points."""
diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py
index 16d08b33150..101405abf50 100644
--- a/litellm/types/proxy/management_endpoints/common_daily_activity.py
+++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py
@@ -26,6 +26,7 @@ class SpendMetrics(BaseModel):
compression_saved_tokens: int = Field(default=0)
compression_savings_spend: float = Field(default=0.0)
prompt_caching_savings_spend: float = Field(default=0.0)
+ gateway_injected_caching_savings_spend: float = Field(default=0.0)
autorouter_savings_spend: float = Field(default=0.0)
total_tokens: int = Field(default=0)
successful_requests: int = Field(default=0)
@@ -88,6 +89,7 @@ class DailySpendMetadata(BaseModel):
total_compression_saved_tokens: int = Field(default=0)
total_compression_savings_spend: float = Field(default=0.0)
total_prompt_caching_savings_spend: float = Field(default=0.0)
+ total_gateway_injected_caching_savings_spend: float = Field(default=0.0)
total_autorouter_savings_spend: float = Field(default=0.0)
page: int = Field(default=1)
total_pages: int = Field(default=1)
@@ -115,6 +117,7 @@ class LiteLLM_DailyUserSpend(BaseModel):
compression_saved_tokens: int = 0
compression_savings_spend: float = 0.0
prompt_caching_savings_spend: float = 0.0
+ gateway_injected_caching_savings_spend: float = 0.0
autorouter_savings_spend: float = 0.0
spend: float = 0.0
api_requests: int = 0
diff --git a/schema.prisma b/schema.prisma
index d9959677116..5582cf930d7 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
@@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
+ gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
index cf3318b32fe..3a736e2a889 100644
--- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
+++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
@@ -2796,3 +2796,116 @@ class TestPromptCacheBreakpointCapability:
def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected):
assert model not in litellm.model_cost
assert supports_openai_prompt_cache_breakpoint(model) is expected
+
+
+class TestRecordGatewayInjection:
+ """The injection marker spend accounting gates prompt-caching savings on."""
+
+ KEY = "litellm_gateway_injected_cache"
+ DEPLOYMENT = "dep-abc"
+
+ def test_records_only_an_actual_injection(self):
+ """A zero delta is hook re-entry and a negative one is a prompt manager replacing
+ the messages; neither is litellm adding a breakpoint."""
+ kwargs: dict = {"metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, 0)
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, -3)
+ assert kwargs["metadata"] == {}
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, 2)
+ assert kwargs["metadata"][self.KEY] == self.DEPLOYMENT
+
+ def test_a_point_this_pass_did_not_place_is_not_claimed(self):
+ """A tool_config point is placed by the Bedrock converse transform, and only when
+ the request carries tools, so its presence here says nothing about whether a
+ breakpoint reaches the wire. Claiming it credited litellm on request shapes that
+ inject nothing, and under-crediting Bedrock tool caching is the fail-closed half.
+ """
+ kwargs: dict = {"metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, 0)
+ assert kwargs["metadata"] == {}
+
+ @pytest.mark.parametrize("kwargs", [{}, {"metadata": None}, {"metadata": "not-a-dict"}])
+ def test_never_introduces_a_metadata_key(self, kwargs):
+ """Stamping must not add a key to a dict the caller splats as ``**kwargs``.
+
+ ``aresponses`` takes ``metadata`` as an explicit parameter and forwards the rest
+ of the request as ``**kwargs``, so a bucket created here arrives twice and the
+ call dies with "got multiple values for keyword argument 'metadata'". Only the
+ proxy reads this marker and it always seeds the bucket first, so a request
+ without one has nothing to record.
+ """
+ before = dict(kwargs)
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, 3)
+ assert kwargs == before
+
+ def test_a_later_pass_cannot_unset_an_earlier_injection(self):
+ kwargs: dict = {"litellm_metadata": {"user_api_key": "k"}, "model_info": {"id": self.DEPLOYMENT}}
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, 2)
+ AnthropicCacheControlHook.record_gateway_injection(kwargs, 0)
+ assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
+
+ def test_v1_messages_auto_injection_stamps_the_marker(self, monkeypatch):
+ monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
+ kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
+ result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control(
+ [{"role": "user", "content": "latest turn"}],
+ "a long system prompt",
+ kwargs,
+ model="claude-sonnet-4-5",
+ custom_llm_provider="anthropic",
+ )
+ assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
+
+ def test_v1_messages_stand_down_leaves_no_marker(self, monkeypatch):
+ """Client-supplied cache_control means the gateway did nothing to credit."""
+ monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
+ kwargs: dict = {"litellm_metadata": {}}
+ AnthropicCacheControlHook.maybe_inject_cache_control(
+ [
+ {
+ "role": "system",
+ "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}],
+ },
+ {"role": "user", "content": "latest turn"},
+ ],
+ None,
+ kwargs,
+ model="claude-sonnet-4-5",
+ custom_llm_provider="anthropic",
+ )
+ assert self.KEY not in kwargs["litellm_metadata"]
+
+ def test_v1_messages_reentry_keeps_the_marker(self, monkeypatch):
+ """A second pass over already-injected messages computes a zero delta, which must
+ leave the first pass's mark standing rather than reading as no injection."""
+ monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
+ kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}}
+ messages = [{"role": "user", "content": "latest turn"}]
+ first_msgs, first_sys = AnthropicCacheControlHook.maybe_inject_cache_control(
+ messages, "a long system prompt", kwargs, model="claude-sonnet-4-5", custom_llm_provider="anthropic"
+ )
+ AnthropicCacheControlHook.maybe_inject_cache_control(
+ first_msgs, first_sys, kwargs, model="claude-sonnet-4-5", custom_llm_provider="anthropic"
+ )
+ assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
+
+ def test_configured_points_skipping_a_marked_target_record_nothing(self):
+ """Configured injection stands down on client breakpoints, so no marker lands."""
+ kwargs: dict = {
+ "litellm_metadata": {},
+ "cache_control_injection_points": [{"location": "message", "role": "system", "index": None}],
+ }
+ AnthropicCacheControlHook.maybe_inject_cache_control(
+ [
+ {
+ "role": "system",
+ "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}],
+ },
+ {"role": "user", "content": "hi"},
+ ],
+ None,
+ kwargs,
+ model="claude-sonnet-4-5",
+ custom_llm_provider="anthropic",
+ )
+ assert self.KEY not in kwargs["litellm_metadata"]
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 29b283ec009..daa5dba3d7b 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -273,9 +273,7 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata():
assert cost is not None, "Cost should not be None"
expected_cost = (10 * custom_input_cost) + (5 * custom_output_cost)
- assert cost == pytest.approx(
- expected_cost
- ), f"Expected {expected_cost}, got {cost}"
+ assert cost == pytest.approx(expected_cost), f"Expected {expected_cost}, got {cost}"
finally:
litellm.model_cost.pop(custom_model_id, None)
@@ -872,13 +870,8 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch):
# Regression check: we expect a distinct DataDogLogger, not the LLM Obs logger
assert type(datadog_logger) is DataDogLogger
- assert any(
- isinstance(cb, DataDogLLMObsLogger)
- for cb in logging_module._in_memory_loggers
- )
- assert any(
- type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers
- )
+ assert any(isinstance(cb, DataDogLLMObsLogger) for cb in logging_module._in_memory_loggers)
+ assert any(type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers)
finally:
logging_module._in_memory_loggers.clear()
@@ -889,9 +882,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
# Required env vars for Logfire integration
monkeypatch.setenv("LOGFIRE_TOKEN", "test-token")
- monkeypatch.setenv(
- "LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev"
- ) # no trailing slash on purpose
+ monkeypatch.setenv("LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev") # no trailing slash on purpose
# Import after env vars are set (important if module-level caching exists)
from litellm.integrations.opentelemetry import OpenTelemetry # logger class
@@ -910,9 +901,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
# Sanity: we got the right logger type and it is cached
assert type(logger) is OpenTelemetry
- assert any(
- type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers
- )
+ assert any(type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers)
# Core regression check: base URL env var should influence the exporter endpoint.
#
@@ -923,9 +912,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
or getattr(logger, "config", None)
or getattr(logger, "_otel_config", None)
)
- assert (
- cfg is not None
- ), "Expected OpenTelemetry logger to keep an otel config on the instance"
+ assert cfg is not None, "Expected OpenTelemetry logger to keep an otel config on the instance"
endpoint = getattr(cfg, "endpoint", None) or getattr(cfg, "otlp_endpoint", None)
assert endpoint is not None, "Expected otel config to expose the OTLP endpoint"
@@ -1083,9 +1070,7 @@ async def test_logging_non_streaming_request():
# Use the filtered call for assertions
call_args = calls_with_expected_input[0]
- standard_logging_object = call_args.kwargs["kwargs"][
- "standard_logging_object"
- ]
+ standard_logging_object = call_args.kwargs["kwargs"]["standard_logging_object"]
assert standard_logging_object["stream"] is not True
finally:
# Restore original callbacks to ensure test isolation
@@ -1103,18 +1088,14 @@ async def test_logging_non_streaming_request():
"agenerate_content_stream",
],
)
-def test_success_handler_skips_sync_callbacks_for_async_requests(
- logging_obj, async_flag
-):
+def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, async_flag):
"""Ensure sync success callbacks are skipped when async call type flags are set."""
from litellm.integrations.custom_logger import CustomLogger
class DummyLogger(CustomLogger):
pass
- logging_obj.stream = (
- False # simulate non-streaming request where sync callbacks would normally run
- )
+ logging_obj.stream = False # simulate non-streaming request where sync callbacks would normally run
logging_obj.model_call_details["litellm_params"] = {async_flag: True}
logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"]
@@ -1190,21 +1171,11 @@ def test_success_handler_runs_sync_callbacks_for_sync_requests(logging_obj, call
def test_is_sync_litellm_request():
assert LitellmLogging._is_sync_litellm_request({}) is True
assert LitellmLogging._is_sync_litellm_request({"acompletion": True}) is False
- assert (
- LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True})
- is False
- )
- assert (
- LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False
- )
+ assert LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True}) is False
+ assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False
assert LitellmLogging._is_sync_litellm_request({"agenerate_content": True}) is False
- assert (
- LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True})
- is False
- )
- assert (
- LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True
- )
+ assert LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True}) is False
+ assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True
def test_get_litellm_params_propagates_allm_passthrough_route():
@@ -1251,9 +1222,7 @@ async def test_dispatch_success_handlers_invokes_callbacks_once_for_final_stream
logging_obj.model_call_details["litellm_params"] = {"acompletion": True}
with (
- patch.object(
- mock_callback, "async_log_success_event", new_callable=AsyncMock
- ) as mock_async_log,
+ patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
patch.object(mock_callback, "log_success_event") as mock_sync_log,
patch.object(
logging_obj,
@@ -1314,9 +1283,7 @@ async def test_dispatch_success_handlers_sync_path_invokes_callback_once_for_fin
with (
patch.object(mock_callback, "log_success_event") as mock_sync_log,
- patch.object(
- mock_callback, "async_log_success_event", new_callable=AsyncMock
- ) as mock_async_log,
+ patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
patch.object(
logging_obj,
"_success_handler_helper_fn",
@@ -1358,20 +1325,14 @@ async def test_dispatch_prefer_async_handlers_runs_legacy_callbacks(
logging_obj.model_call_details["litellm_params"] = {}
with (
- patch.object(
- logging_obj, "async_success_handler", new_callable=AsyncMock
- ) as mock_async,
- patch.object(
- logging_obj, "success_handler", new_callable=MagicMock
- ) as mock_sync,
+ patch.object(logging_obj, "async_success_handler", new_callable=AsyncMock) as mock_async,
+ patch.object(logging_obj, "success_handler", new_callable=MagicMock) as mock_sync,
patch.object(
logging_obj,
"_should_run_sync_callbacks_for_async_calls",
return_value=True,
),
- patch(
- "litellm.litellm_core_utils.litellm_logging.executor.submit"
- ) as mock_submit,
+ patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_success_handlers(
result=result,
@@ -1405,9 +1366,7 @@ async def test_dispatch_success_handlers_invokes_async_callback_for_pass_through
try:
with (
- patch.object(
- mock_callback, "async_log_success_event", new_callable=AsyncMock
- ) as mock_async_log,
+ patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
patch.object(mock_callback, "log_success_event") as mock_sync_log,
):
await logging_obj.dispatch_success_handlers(result={"id": "pt-1"})
@@ -1434,20 +1393,14 @@ async def test_dispatch_failure_handlers_prefer_async_does_not_submit_sync_handl
logging_obj.model_call_details["litellm_params"] = {}
with (
- patch.object(
- logging_obj, "async_failure_handler", new_callable=AsyncMock
- ) as mock_async,
- patch.object(
- logging_obj, "failure_handler", new_callable=MagicMock
- ) as mock_sync,
+ patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async,
+ patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
patch.object(
logging_obj,
"_should_run_sync_failure_callbacks_for_async_calls",
return_value=False,
),
- patch(
- "litellm.litellm_core_utils.litellm_logging.executor.submit"
- ) as mock_submit,
+ patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_failure_handlers(
exception,
@@ -1530,12 +1483,8 @@ async def test_dispatch_failure_handlers_submits_sync_handler_for_failure_only_c
patch.object(litellm, "success_callback", []),
patch.object(litellm, "failure_callback", [_sync_failure_callback]),
patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock),
- patch.object(
- logging_obj, "failure_handler", new_callable=MagicMock
- ) as mock_sync,
- patch(
- "litellm.litellm_core_utils.litellm_logging.executor.submit"
- ) as mock_submit,
+ patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
+ patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_failure_handlers(
exception,
@@ -1562,15 +1511,9 @@ async def test_dispatch_failure_handlers_sync_sdk_shortcut_runs_sync_handler_inl
logging_obj.model_call_details["litellm_params"] = {}
with (
- patch.object(
- logging_obj, "async_failure_handler", new_callable=AsyncMock
- ) as mock_async,
- patch.object(
- logging_obj, "failure_handler", new_callable=MagicMock
- ) as mock_sync,
- patch(
- "litellm.litellm_core_utils.litellm_logging.executor.submit"
- ) as mock_submit,
+ patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async,
+ patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
+ patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
):
await logging_obj.dispatch_failure_handlers(
exception,
@@ -1617,14 +1560,10 @@ def test_success_handler_skips_guardrail_logging_hook_when_disabled(logging_obj)
event_hook=GuardrailEventHooks.logging_only,
)
guardrail.should_run_guardrail = MagicMock(return_value=False)
- guardrail.logging_hook = MagicMock(
- return_value=(logging_obj.model_call_details, model_response)
- )
+ guardrail.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response))
dummy_logger = DummyLogger()
- dummy_logger.logging_hook = MagicMock(
- return_value=(logging_obj.model_call_details, model_response)
- )
+ dummy_logger.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response))
with patch.object(
logging_obj,
@@ -1758,11 +1697,7 @@ def test_get_request_tags_from_metadata_and_litellm_metadata():
# Test case 2: Tags in litellm_metadata only
tags = StandardLoggingPayloadSetup._get_request_tags(
- litellm_params={
- "litellm_metadata": {
- "tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]
- }
- },
+ litellm_params={"litellm_metadata": {"tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]}},
proxy_server_request={},
)
assert "litellm-metadata-tag-1" in tags
@@ -1867,15 +1802,9 @@ def test_get_request_tags_does_not_mutate_original_tags():
user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")])
user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")])
- assert (
- user_agent_count_1 == 2
- ), f"Expected 2 User-Agent tags, got {user_agent_count_1}"
- assert (
- user_agent_count_2 == 2
- ), f"Expected 2 User-Agent tags, got {user_agent_count_2}"
- assert (
- user_agent_count_3 == 2
- ), f"Expected 2 User-Agent tags, got {user_agent_count_3}"
+ assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}"
+ assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}"
+ assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}"
# Verify all returned lists are independent (different objects)
assert tags1 is not tags2
@@ -1908,9 +1837,7 @@ def test_get_extra_header_tags():
# Test case 3: Extra headers configured but request has no headers dict
litellm.extra_spend_tag_headers = ["x-custom", "x-tenant"]
- result = StandardLoggingPayloadSetup._get_extra_header_tags(
- proxy_server_request={"headers": "not-a-dict"}
- )
+ result = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": "not-a-dict"})
assert result is None
# Test case 4: Extra headers configured but none match request headers
@@ -2211,9 +2138,7 @@ def test_get_masked_values():
"presidio_anonymizer_api_base": None,
"vertex_credentials": "{sensitive_api_key}",
}
- masked_values = _get_masked_values(
- sensitive_object, unmasked_length=4, number_of_asterisks=4
- )
+ masked_values = _get_masked_values(sensitive_object, unmasked_length=4, number_of_asterisks=4)
assert masked_values["presidio_anonymizer_api_base"] is None
assert masked_values["vertex_credentials"] == "{s****y}"
@@ -2238,9 +2163,7 @@ async def test_e2e_generate_cold_storage_object_key_successful():
patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
):
# Mock the S3 object key generation to return a predictable result
- mock_get_s3_key.return_value = (
- "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
- )
+ mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
# Call the function
result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@@ -2281,16 +2204,12 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
with (
patch("litellm.cold_storage_custom_logger", "s3_v2"),
- patch(
- "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name"
- ) as mock_get_logger,
+ patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger,
patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
):
# Setup mocks
mock_get_logger.return_value = mock_custom_logger
- mock_get_s3_key.return_value = (
- "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
- )
+ mock_get_s3_key.return_value = "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
# Call the function
result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@@ -2309,9 +2228,7 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
)
# Verify the result
- assert (
- result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
- )
+ assert result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
@pytest.mark.asyncio
@@ -2334,16 +2251,12 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path():
with (
patch("litellm.cold_storage_custom_logger", "s3_v2"),
- patch(
- "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name"
- ) as mock_get_logger,
+ patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger,
patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
):
# Setup mocks
mock_get_logger.return_value = mock_custom_logger
- mock_get_s3_key.return_value = (
- "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
- )
+ mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
# Call the function
result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@@ -2459,9 +2372,7 @@ def test_get_usage_as_dict():
assert result == {"prompt_tokens": 20, "completion_tokens": 30}
# Test case 5: response_obj with no usage key returns empty
- result = StandardLoggingPayloadSetup.get_usage_as_dict(
- response_obj={"id": "resp-1", "choices": []}
- )
+ result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj={"id": "resp-1", "choices": []})
assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
@@ -2474,26 +2385,20 @@ def test_append_system_prompt_messages():
# Test case 1: system in kwargs with existing messages
kwargs = {"system": "You are a helpful assistant"}
messages = [{"role": "user", "content": "Hello"}]
- result = StandardLoggingPayloadSetup.append_system_prompt_messages(
- kwargs=kwargs, messages=messages
- )
+ result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
assert len(result) == 2
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
assert result[1] == {"role": "user", "content": "Hello"}
# Test case 2: system in kwargs with None messages
kwargs = {"system": "You are a helpful assistant"}
- result = StandardLoggingPayloadSetup.append_system_prompt_messages(
- kwargs=kwargs, messages=None
- )
+ result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=None)
assert len(result) == 1
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
# Test case 3: system in kwargs with empty messages list
kwargs = {"system": "You are a helpful assistant"}
- result = StandardLoggingPayloadSetup.append_system_prompt_messages(
- kwargs=kwargs, messages=[]
- )
+ result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=[])
assert len(result) == 1
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
@@ -2503,24 +2408,18 @@ def test_append_system_prompt_messages():
{"role": "system", "content": "You are a helpful assistant"},
{"role": "user", "content": "Hello"},
]
- result = StandardLoggingPayloadSetup.append_system_prompt_messages(
- kwargs=kwargs, messages=messages
- )
+ result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
assert len(result) == 2
assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
# Test case 5: no system in kwargs returns messages unchanged
kwargs = {}
messages = [{"role": "user", "content": "Hello"}]
- result = StandardLoggingPayloadSetup.append_system_prompt_messages(
- kwargs=kwargs, messages=messages
- )
+ result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
assert result == messages
# Test case 6: None kwargs returns messages unchanged
- result = StandardLoggingPayloadSetup.append_system_prompt_messages(
- kwargs=None, messages=messages
- )
+ result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=None, messages=messages)
assert result == messages
@@ -2581,12 +2480,11 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu
# Verify that standard_logging_object was set
assert "standard_logging_object" in logging_obj.model_call_details, (
- "standard_logging_object should be set for pass-through endpoints "
- "even when complete_streaming_response is None"
+ "standard_logging_object should be set for pass-through endpoints even when complete_streaming_response is None"
+ )
+ assert logging_obj.model_call_details["standard_logging_object"] is not None, (
+ "standard_logging_object should not be None for pass-through endpoints"
)
- assert (
- logging_obj.model_call_details["standard_logging_object"] is not None
- ), "standard_logging_object should not be None for pass-through endpoints"
# Verify that async_complete_streaming_response was set to prevent re-processing
# This is consistent with the existing code pattern for regular streaming
@@ -2594,15 +2492,13 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu
"async_complete_streaming_response should be set to prevent re-processing, "
"consistent with the existing code pattern"
)
- assert (
- logging_obj.model_call_details["async_complete_streaming_response"] is result
- ), "async_complete_streaming_response should be set to the result"
+ assert logging_obj.model_call_details["async_complete_streaming_response"] is result, (
+ "async_complete_streaming_response should be set to the result"
+ )
# Verify that response_cost is set to None (cost calculation not possible for pass-through)
# This is consistent with the error handling in the non-pass-through code path
- assert (
- "response_cost" in logging_obj.model_call_details
- ), "response_cost should be set for pass-through endpoints"
+ assert "response_cost" in logging_obj.model_call_details, "response_cost should be set for pass-through endpoints"
assert logging_obj.model_call_details["response_cost"] is None, (
"response_cost should be None for pass-through endpoints since "
"StandardPassThroughResponseObject doesn't have standard usage info"
@@ -2661,14 +2557,10 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp
# Verify first call set the values
assert "standard_logging_object" in logging_obj.model_call_details
assert "async_complete_streaming_response" in logging_obj.model_call_details
- first_standard_logging_object = logging_obj.model_call_details[
- "standard_logging_object"
- ]
+ first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"]
# Second call - should return early due to async_complete_streaming_response guard
- with patch.object(
- logging_obj, "get_combined_callback_list", return_value=[]
- ) as mock_callbacks:
+ with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks:
await logging_obj.async_success_handler(
result=result,
start_time=start_time,
@@ -2679,10 +2571,9 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp
mock_callbacks.assert_not_called()
# Verify standard_logging_object wasn't modified by second call
- assert (
- logging_obj.model_call_details["standard_logging_object"]
- is first_standard_logging_object
- ), "standard_logging_object should not be modified on re-processing"
+ assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, (
+ "standard_logging_object should not be modified on re-processing"
+ )
@pytest.mark.asyncio
@@ -2721,9 +2612,7 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_
}
# Create a pass-through response object (simulating unparseable streaming response)
- result = StandardPassThroughResponseObject(
- response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]'
- )
+ result = StandardPassThroughResponseObject(response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]')
start_time = datetime.now()
end_time = datetime.now()
@@ -2743,9 +2632,9 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_
"standard_logging_object should be set for streaming pass-through endpoints "
"even when the response cannot be parsed into a ModelResponse"
)
- assert (
- logging_obj.model_call_details["standard_logging_object"] is not None
- ), "standard_logging_object should not be None for streaming pass-through endpoints"
+ assert logging_obj.model_call_details["standard_logging_object"] is not None, (
+ "standard_logging_object should not be None for streaming pass-through endpoints"
+ )
def test_get_error_information_error_code_priority():
@@ -2787,30 +2676,22 @@ def test_get_error_information_error_code_priority():
self.message = message
super().__init__(message)
- both_exception = BothAttributesException(
- code="400", status_code=500, message="Bad Request"
- )
+ both_exception = BothAttributesException(code="400", status_code=500, message="Bad Request")
result = StandardLoggingPayloadSetup.get_error_information(both_exception)
assert result["error_code"] == "400" # Should prefer 'code' over 'status_code'
# Test case 4: Exception with 'code' as empty string - should fall back to 'status_code'
- empty_code_exception = BothAttributesException(
- code="", status_code=404, message="Not Found"
- )
+ empty_code_exception = BothAttributesException(code="", status_code=404, message="Not Found")
result = StandardLoggingPayloadSetup.get_error_information(empty_code_exception)
assert result["error_code"] == "404" # Should fall back to status_code
# Test case 5: Exception with 'code' as "None" string - should fall back to 'status_code'
- none_string_exception = BothAttributesException(
- code="None", status_code=503, message="Service Unavailable"
- )
+ none_string_exception = BothAttributesException(code="None", status_code=503, message="Service Unavailable")
result = StandardLoggingPayloadSetup.get_error_information(none_string_exception)
assert result["error_code"] == "503" # Should fall back to status_code
# Test case 6: Exception with 'code' as None - should fall back to 'status_code'
- none_code_exception = BothAttributesException(
- code=None, status_code=401, message="Unauthorized"
- )
+ none_code_exception = BothAttributesException(code=None, status_code=401, message="Unauthorized")
result = StandardLoggingPayloadSetup.get_error_information(none_code_exception)
assert result["error_code"] == "401" # Should fall back to status_code
@@ -2859,9 +2740,7 @@ def test_get_error_information_prefers_message_attribute_over_str():
)
result = StandardLoggingPayloadSetup.get_error_information(exc)
- assert (
- result["error_message"] == msg
- ), f"expected message from .message attribute, got {result['error_message']!r}"
+ assert result["error_message"] == msg, f"expected message from .message attribute, got {result['error_message']!r}"
assert result["error_code"] == "401"
assert result["error_class"] == "ProxyExceptionLike"
@@ -2936,8 +2815,7 @@ def test_get_error_information_preserves_explicit_empty_message():
exc = ProxyExceptionLike(message="", code=500)
result = StandardLoggingPayloadSetup.get_error_information(exc)
assert result["error_message"] == "", (
- "explicit empty .message must survive verbatim; got "
- f"{result['error_message']!r}"
+ f"explicit empty .message must survive verbatim; got {result['error_message']!r}"
)
@@ -3200,9 +3078,7 @@ def test_process_hidden_params_recalculates_cost_after_failure_handler_zero():
choices=[{"message": {"role": "assistant", "content": "ok"}}],
usage=Usage(prompt_tokens=9698, completion_tokens=30, total_tokens=9728),
)
- logging_obj._process_hidden_params_and_response_cost(
- result, datetime.now(), datetime.now()
- )
+ logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
cost = logging_obj.model_call_details.get("response_cost")
assert cost is not None and cost > 0
@@ -3226,9 +3102,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params():
litellm_call_id="test-hidden-zero-cost",
function_id="test-hidden-zero-cost",
)
- logging_obj.model_call_details["litellm_params"] = {
- "model": "gemini-2.5-flash-lite"
- }
+ logging_obj.model_call_details["litellm_params"] = {"model": "gemini-2.5-flash-lite"}
logging_obj.optional_params = {}
result = ModelResponse(
@@ -3238,9 +3112,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params():
)
result._hidden_params = {"response_cost": 0.0}
- logging_obj._process_hidden_params_and_response_cost(
- result, datetime.now(), datetime.now()
- )
+ logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
assert logging_obj.model_call_details.get("response_cost") == 0.0
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
@@ -3289,9 +3161,7 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer
)
result._hidden_params = {"response_cost": passthrough_cost}
- logging_obj._process_hidden_params_and_response_cost(
- result, datetime.now(), datetime.now()
- )
+ logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
assert logging_obj.model_call_details.get("response_cost") == passthrough_cost
slo = logging_obj.model_call_details.get("standard_logging_object") or {}
@@ -3348,9 +3218,7 @@ def test_function_setup_litellm_metadata_populates_metadata():
assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash
# metadata should be a COPY, not an alias — mutating one must not affect the other
- assert (
- metadata is not litellm_metadata
- ), "litellm_params['metadata'] should be a copy, not the same object"
+ assert metadata is not litellm_metadata, "litellm_params['metadata'] should be a copy, not the same object"
def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
@@ -3395,9 +3263,9 @@ def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
litellm_params = logging_obj.model_call_details.get("litellm_params", {})
litellm_metadata = litellm_params.get("litellm_metadata")
assert litellm_metadata is not None
- assert litellm_metadata.get("standard_logging_guardrail_information") == [
- guardrail_entry
- ], "guardrail writes after function_setup must be visible to the logging object"
+ assert litellm_metadata.get("standard_logging_guardrail_information") == [guardrail_entry], (
+ "guardrail writes after function_setup must be visible to the logging object"
+ )
assert litellm_metadata.get("applied_guardrails") == ["pam-ethical-request"]
merged = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)
@@ -3566,9 +3434,7 @@ def test_failure_handler_skips_sync_callbacks_for_pass_through_requests(logging_
@pytest.mark.parametrize("call_type", ["completion", "acompletion"])
-def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(
- logging_obj, call_type
-):
+def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(logging_obj, call_type):
"""Ensure sync failure callbacks still fire for normal (non-pass-through) requests."""
from litellm.integrations.custom_logger import CustomLogger
@@ -3729,9 +3595,7 @@ def test_standard_logging_hidden_params_backfills_response_cost_without_mutating
)
response._hidden_params = {"response_cost": None, "model_id": "mid-test"}
- payload = logging_obj._build_standard_logging_payload(
- response, datetime.now(), datetime.now()
- )
+ payload = logging_obj._build_standard_logging_payload(response, datetime.now(), datetime.now())
assert payload is not None
assert payload["hidden_params"]["response_cost"] == 0.002
@@ -3785,10 +3649,7 @@ def test_merge_hidden_params_from_response_into_metadata_no_op_when_empty():
_hidden_params = {}
logging_obj._merge_hidden_params_from_response_into_metadata(_NoHp())
- assert (
- "hidden_params"
- not in logging_obj.model_call_details["litellm_params"]["metadata"]
- )
+ assert "hidden_params" not in logging_obj.model_call_details["litellm_params"]["metadata"]
# ── StandardLoggingPayloadSetup.get_additional_headers ───────────────────────
@@ -3985,9 +3846,7 @@ def test_success_handler_computes_cost_for_dict_response():
"_build_standard_logging_payload",
return_value={"response_cost": expected_cost},
),
- patch(
- "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
- ),
+ patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(
logging_obj,
"_is_recognized_call_type_for_logging",
@@ -4024,9 +3883,7 @@ def test_success_handler_preserves_precomputed_cost_for_dict_response():
"_build_standard_logging_payload",
return_value={"response_cost": precomputed_cost},
),
- patch(
- "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
- ),
+ patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(
logging_obj,
"_is_recognized_call_type_for_logging",
@@ -4065,9 +3922,7 @@ def test_success_handler_unified_helper_runs_for_typed_results():
"_build_standard_logging_payload",
return_value={"response_cost": expected_cost},
),
- patch(
- "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
- ),
+ patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
patch.object(
logging_obj,
"_is_recognized_call_type_for_logging",
@@ -4122,9 +3977,7 @@ class TestFirstApiCallStartTimeSetOnce:
assert first == obj.model_call_details["api_call_start_time"]
# Set on the logging object only — user metadata untouched.
assert user_meta == {}
- assert (
- "first_api_call_start_time" not in obj.model_call_details["litellm_params"]
- )
+ assert "first_api_call_start_time" not in obj.model_call_details["litellm_params"]
time.sleep(0.002) # ensure a distinct retry timestamp
obj.pre_call(input="hi", api_key="sk-test")
@@ -4141,18 +3994,16 @@ def test_get_error_information_for_logging_payload_ignores_spoofed_disconnect_wi
baseline = StandardLoggingPayloadSetup.get_error_information(
original_exception=ValueError("provider failure"),
)
- error_information, error_str = (
- StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
- metadata={
- "error_information": {
- "error_code": "499",
- "error_message": "Client disconnected the request",
- "error_class": "ClientDisconnected",
- }
- },
- original_exception=ValueError("provider failure"),
- error_str="provider failure",
- )
+ error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+ metadata={
+ "error_information": {
+ "error_code": "499",
+ "error_message": "Client disconnected the request",
+ "error_class": "ClientDisconnected",
+ }
+ },
+ original_exception=ValueError("provider failure"),
+ error_str="provider failure",
)
assert error_information == baseline
assert error_str == "provider failure"
@@ -4166,22 +4017,18 @@ def test_get_error_information_for_logging_payload_client_disconnect():
"error_message": "Client disconnected the request",
"error_class": "ClientDisconnected",
}
- error_information, error_str = (
- StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
- metadata={"client_disconnected": True, "error_information": custom_error},
- original_exception=None,
- error_str=None,
- )
+ error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+ metadata={"client_disconnected": True, "error_information": custom_error},
+ original_exception=None,
+ error_str=None,
)
assert error_information == custom_error
assert error_str == "Client disconnected the request"
- error_information, error_str = (
- StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
- metadata={"client_disconnected": True},
- original_exception=None,
- error_str="existing error",
- )
+ error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+ metadata={"client_disconnected": True},
+ original_exception=None,
+ error_str="existing error",
)
assert error_information["error_code"] == "499"
assert error_str == "existing error"
@@ -4189,12 +4036,10 @@ def test_get_error_information_for_logging_payload_client_disconnect():
baseline = StandardLoggingPayloadSetup.get_error_information(
original_exception=None,
)
- error_information, error_str = (
- StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
- metadata={},
- original_exception=None,
- error_str=None,
- )
+ error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+ metadata={},
+ original_exception=None,
+ error_str=None,
)
assert error_information == baseline
assert error_str is None
@@ -4229,9 +4074,7 @@ def test_get_error_information_prefers_message_attribute_over_empty_str():
def __str__(self):
return ""
- info = StandardLoggingPayloadSetup.get_error_information(
- original_exception=_SilentExc()
- )
+ info = StandardLoggingPayloadSetup.get_error_information(original_exception=_SilentExc())
assert info["error_message"] == "real failure detail"
assert info["error_code"] == "401"
@@ -4262,9 +4105,7 @@ def _responses_api_response_with_text(text="hello world"):
type="message",
role="assistant",
status="completed",
- content=[
- ResponseOutputText(annotations=[], text=text, type="output_text")
- ],
+ content=[ResponseOutputText(annotations=[], text=text, type="output_text")],
)
],
usage=ResponseAPIUsage(input_tokens=11, output_tokens=7, total_tokens=18),
@@ -4279,9 +4120,7 @@ def _responses_api_response_with_text(text="hello world"):
("ResponseFailedEvent", "response.failed"),
],
)
-def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(
- event_cls, event_type
-):
+def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(event_cls, event_type):
"""Regression for #28595 / #28943. When anthropic_messages routes to the OpenAI
Responses backend and stream=True, success_handler receives a terminal Responses
API event. The handler must translate it to a ModelResponse whose choices carry
@@ -4320,10 +4159,7 @@ def test_handle_anthropic_messages_response_logging_passes_model_response_throug
"""Anthropic-native path already yields a ModelResponse; it must be returned unchanged."""
logging_obj = _anthropic_messages_logging_obj()
model_response = ModelResponse()
- assert (
- logging_obj._handle_anthropic_messages_response_logging(result=model_response)
- is model_response
- )
+ assert logging_obj._handle_anthropic_messages_response_logging(result=model_response) is model_response
def test_handle_anthropic_messages_response_logging_degrades_on_unparseable_responses_payload():
@@ -4619,9 +4455,7 @@ def test_non_image_response_has_no_output_image_count(logging_obj):
def test_zero_token_video_usage_preserves_duration_seconds(logging_obj):
"""Video usage bills by duration; the payload must keep duration_seconds even with zero tokens."""
- payload = _build_payload_for_media_response(
- logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}}
- )
+ payload = _build_payload_for_media_response(logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}})
assert payload is not None
assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0
@@ -5782,3 +5616,93 @@ def test_failure_handler_helper_fn_builds_payload_once_per_exception():
other_exc = _raise_and_catch(_ClientError(status_code=429, message="rate limited"))
obj._failure_handler_helper_fn(exception=other_exc, traceback_exception="")
assert obj.model_call_details["standard_logging_object"] is not first_payload
+
+
+@pytest.mark.asyncio
+async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_obj):
+ """The savings gate reads litellm_gateway_injected_cache from the request's
+ metadata bucket. Recording lives in the shared prompt-hook wrappers, so chat,
+ /v1/responses, router prompt deployments, and proxy prompt templates all mark
+ injected requests the same way; a hook that injects nothing leaves no marker."""
+ from litellm.integrations.custom_prompt_management import CustomPromptManagement
+
+ class _InjectingHook(CustomPromptManagement):
+ def get_chat_completion_prompt(
+ self,
+ model,
+ messages,
+ non_default_params,
+ prompt_id,
+ prompt_variables,
+ dynamic_callback_params,
+ prompt_label=None,
+ prompt_version=None,
+ prompt_spec=None,
+ ):
+ marked = [{**messages[0], "cache_control": {"type": "ephemeral"}}, *messages[1:]]
+ return model, marked, non_default_params
+
+ async def async_get_chat_completion_prompt(
+ self,
+ model,
+ messages,
+ non_default_params,
+ prompt_id,
+ prompt_variables,
+ dynamic_callback_params,
+ litellm_logging_obj=None,
+ tools=None,
+ prompt_label=None,
+ prompt_version=None,
+ prompt_spec=None,
+ ):
+ return self.get_chat_completion_prompt(
+ model, messages, non_default_params, prompt_id, prompt_variables, dynamic_callback_params
+ )
+
+ class _PassthroughHook(CustomPromptManagement):
+ def get_chat_completion_prompt(
+ self,
+ model,
+ messages,
+ non_default_params,
+ prompt_id,
+ prompt_variables,
+ dynamic_callback_params,
+ prompt_label=None,
+ prompt_version=None,
+ prompt_spec=None,
+ ):
+ return model, messages, non_default_params
+
+ request_kwargs = {"metadata": {}, "model_info": {"id": "dep-of-this-attempt"}}
+ _, marked, _ = await logging_obj.async_get_chat_completion_prompt(
+ model="claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ non_default_params={},
+ prompt_variables=None,
+ prompt_management_logger=_InjectingHook(),
+ request_kwargs=request_kwargs,
+ )
+ assert request_kwargs["metadata"]["litellm_gateway_injected_cache"] == "dep-of-this-attempt"
+
+ logging_obj.get_chat_completion_prompt(
+ model="claude-sonnet-5",
+ messages=marked,
+ non_default_params={},
+ prompt_variables=None,
+ prompt_management_logger=_PassthroughHook(),
+ request_kwargs=request_kwargs,
+ )
+ assert request_kwargs["metadata"]["litellm_gateway_injected_cache"] == "dep-of-this-attempt"
+
+ untouched = {"metadata": {}}
+ logging_obj.get_chat_completion_prompt(
+ model="claude-sonnet-5",
+ messages=[{"role": "user", "content": "hi"}],
+ non_default_params={},
+ prompt_variables=None,
+ prompt_management_logger=_PassthroughHook(),
+ request_kwargs=untouched,
+ )
+ assert "litellm_gateway_injected_cache" not in untouched["metadata"]
diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
index a00815345aa..681132105ad 100644
--- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
+++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
@@ -207,6 +207,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"compression_saved_tokens": 0,
"compression_savings_spend": 0,
"prompt_caching_savings_spend": 0,
+ "gateway_injected_caching_savings_spend": 0,
"autorouter_savings_spend": 0,
}
@@ -258,6 +259,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"compression_saved_tokens": 0,
"compression_savings_spend": 0,
"prompt_caching_savings_spend": 0,
+ "gateway_injected_caching_savings_spend": 0,
"autorouter_savings_spend": 0,
}
diff --git a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py
index c2d0f64461a..c1efb3e7220 100644
--- a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py
+++ b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py
@@ -85,10 +85,10 @@ def test_one_statement_carries_every_row_in_the_batch():
assert sql.count("INSERT INTO") == 1
assert len(re.findall(r"ON CONFLICT", sql)) == 1
- # 22 bound columns per row plus the inlined updated_at, so the row count is what
+ # 23 bound columns per row plus the inlined updated_at, so the row count is what
# separates one multi-row statement from a hundred single-row ones.
- assert len(params) == 100 * 22
- assert "$2200::text" in sql
+ assert len(params) == 100 * 23
+ assert "$2300::text" in sql
assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1
diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
index ca1827aa38e..b1f647bdd3d 100644
--- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
+++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
@@ -2331,6 +2331,7 @@ async def test_daily_transaction_carries_compression_saved_tokens():
metadata = {
"usage_object": {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 15},
+ "litellm_gateway_injected_cache": "dep-of-the-compression-row",
"compression_savings": {
"tokens_before": 12000,
"tokens_after": 5000,
@@ -2354,6 +2355,7 @@ async def test_daily_transaction_carries_compression_saved_tokens():
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
+ "model_id": "dep-of-the-compression-row",
"call_type": "anthropic_messages",
"prompt_tokens": 5000,
"completion_tokens": 10,
@@ -2740,3 +2742,105 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_on_responses_calls(
assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}]
assert PrismaClient.spend_log_flush_requested.is_set() is expects_flush
PrismaClient.spend_log_flush_requested.clear()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "injected_deployment, attributed",
+ [
+ pytest.param("dep-of-this-row", True, id="this-deployment-injected"),
+ pytest.param("dep-of-a-sibling-leg", False, id="a-sibling-deployment-injected"),
+ pytest.param("", True, id="injected-before-a-deployment-was-chosen"),
+ ],
+)
+async def test_caching_savings_are_attributed_to_the_deployment_that_was_injected(
+ injected_deployment, attributed
+):
+ """Retries, same-group failover and cross-model-group fallbacks all reuse one metadata
+ bucket and one litellm_call_id, so a marker written by the leg that injected is
+ visible to every sibling and nothing request-scoped can tell them apart.
+
+ Naming the deployment it injected for is what keeps the credit on that leg: a row
+ billed for a different deployment reads it as no injection, so no seam has to strip
+ it and a deployment that injected nothing is never credited for the one that did.
+
+ An injection that ran before any deployment was chosen, which is what the proxy does
+ for prompt templates, is written into the payload every leg goes on to send, so it
+ marks the request for all of them and each leg keeps the credit.
+ """
+ writer = DBSpendUpdateWriter()
+ mock_prisma = MagicMock()
+ mock_prisma.get_request_status = MagicMock(return_value="success")
+
+ payload = {
+ "request_id": "req-fallback-leg",
+ "user": "test-user",
+ "startTime": "2026-07-17T00:00:00",
+ "api_key": "test-key",
+ "model": "claude-sonnet-5",
+ "custom_llm_provider": "anthropic",
+ "model_group": "claude-sonnet-5",
+ "model_id": "dep-of-this-row",
+ "call_type": "anthropic_messages",
+ "prompt_tokens": 5000,
+ "completion_tokens": 10,
+ "spend": 0.05,
+ "metadata": json.dumps(
+ {
+ "usage_object": {"cache_read_input_tokens": 4242, "cache_creation_input_tokens": 1111},
+ "litellm_gateway_injected_cache": injected_deployment,
+ }
+ ),
+ }
+
+ transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
+ payload=payload,
+ prisma_client=mock_prisma,
+ type="user",
+ )
+
+ assert transaction is not None
+ assert transaction["prompt_caching_savings_spend"] != 0.0
+ assert (transaction["gateway_injected_caching_savings_spend"] != 0.0) is attributed
+
+
+@pytest.mark.asyncio
+async def test_daily_transaction_attributes_caching_savings_only_with_an_injection_marker():
+ """Cached usage with no litellm_gateway_injected_cache marker is still a real saving.
+
+ Client-sent cache_control and implicit provider caching leave no marker, so the row
+ keeps the total the customer actually got while the gateway-attributed column stays
+ empty, which is what separates what caching saved from what litellm can claim.
+ """
+ writer = DBSpendUpdateWriter()
+ mock_prisma = MagicMock()
+ mock_prisma.get_request_status = MagicMock(return_value="success")
+
+ payload = {
+ "request_id": "req-ungated-caching",
+ "user": "test-user",
+ "startTime": "2026-07-17T00:00:00",
+ "api_key": "test-key",
+ "model": "claude-sonnet-5",
+ "custom_llm_provider": "anthropic",
+ "model_group": "claude-sonnet-5",
+ "call_type": "anthropic_messages",
+ "prompt_tokens": 5000,
+ "completion_tokens": 10,
+ "spend": 0.05,
+ "metadata": json.dumps(
+ {"usage_object": {"cache_read_input_tokens": 4242, "cache_creation_input_tokens": 1111}}
+ ),
+ }
+
+ transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
+ payload=payload,
+ prisma_client=mock_prisma,
+ type="user",
+ )
+
+ assert transaction is not None
+ assert transaction["cache_read_input_tokens"] == 4242
+ assert transaction["cache_creation_input_tokens"] == 1111
+ assert transaction["prompt_caching_savings_spend"] != 0.0
+ assert transaction["gateway_injected_caching_savings_spend"] == 0.0
diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
index 1bcb331430e..a258127acff 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
@@ -155,6 +155,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
+ "gateway_injected_caching_savings_spend": 0.0,
"autorouter_savings_spend": 0.0,
"failed_requests": 0,
}
@@ -485,6 +486,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_1.compression_saved_tokens = 0
mock_record_1.compression_savings_spend = 0.0
mock_record_1.prompt_caching_savings_spend = 0.0
+ mock_record_1.gateway_injected_caching_savings_spend = 0.0
mock_record_1.autorouter_savings_spend = 0.0
mock_record_1.api_requests = 10
mock_record_1.successful_requests = 9
@@ -508,6 +510,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_2.compression_saved_tokens = 0
mock_record_2.compression_savings_spend = 0.0
mock_record_2.prompt_caching_savings_spend = 0.0
+ mock_record_2.gateway_injected_caching_savings_spend = 0.0
mock_record_2.autorouter_savings_spend = 0.0
mock_record_2.api_requests = 5
mock_record_2.successful_requests = 5
@@ -571,6 +574,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
+ "gateway_injected_caching_savings_spend": 0.0,
"autorouter_savings_spend": 0.0,
"failed_requests": 0,
}
@@ -657,6 +661,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr
compression_saved_tokens=0,
compression_savings_spend=0.0,
prompt_caching_savings_spend=0.0,
+ gateway_injected_caching_savings_spend=0.0,
autorouter_savings_spend=0.0,
api_requests=1,
successful_requests=1,
@@ -1089,6 +1094,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
"compression_saved_tokens": None,
"compression_savings_spend": None,
"prompt_caching_savings_spend": None,
+ "gateway_injected_caching_savings_spend": None,
"autorouter_savings_spend": None,
"api_requests": None,
"successful_requests": None,
@@ -1133,6 +1139,7 @@ def _no_spend_record():
compression_saved_tokens=None,
compression_savings_spend=None,
prompt_caching_savings_spend=None,
+ gateway_injected_caching_savings_spend=None,
autorouter_savings_spend=None,
api_requests=None,
successful_requests=None,
@@ -1242,6 +1249,7 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost=
compression_saved_tokens=0,
compression_savings_spend=0,
prompt_caching_savings_spend=0,
+ gateway_injected_caching_savings_spend=0,
autorouter_savings_spend=0,
total_tokens=0,
api_requests=0,
@@ -1307,6 +1315,7 @@ def _grouping_row(
compression_saved_tokens=0,
compression_savings_spend=0.0,
prompt_caching_savings_spend=0.0,
+ gateway_injected_caching_savings_spend=0.0,
autorouter_savings_spend=0.0,
api_requests=0,
successful_requests=0,
@@ -1466,6 +1475,7 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib
compression_saved_tokens=0,
compression_savings_spend=0,
prompt_caching_savings_spend=0,
+ gateway_injected_caching_savings_spend=0,
autorouter_savings_spend=0,
total_tokens=0,
api_requests=0,
@@ -1869,6 +1879,7 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown():
"compression_saved_tokens": 0,
"compression_savings_spend": 0.0,
"prompt_caching_savings_spend": 0.0,
+ "gateway_injected_caching_savings_spend": 0.0,
"autorouter_savings_spend": 0.0,
"failed_requests": 0,
"prompt_tokens": 0,
diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py
index bb8345a9142..bc9e2a0f7cc 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_savings.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py
@@ -8,6 +8,7 @@ from litellm.proxy.spend_tracking.savings import (
_baseline_usage,
compute_autorouter_savings,
compute_savings_spend,
+ marks_gateway_injection,
)
from litellm.router import Router
from litellm.types.utils import Usage
@@ -59,6 +60,7 @@ def test_compression_savings_priced_at_input_rate():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=4389,
+ gateway_injected_cache=True,
)
assert result.compression == pytest.approx(4389 * input_cost)
assert result.compression > 0
@@ -74,6 +76,7 @@ def test_prompt_caching_savings_priced_at_input_minus_cache_read():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 8200},
)
assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost))
@@ -126,6 +129,7 @@ def test_prompt_caching_savings_nets_out_the_cache_write_premium():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=usage_object,
)
assert result.prompt_caching == pytest.approx(_net_caching_savings_against_biller(usage_object))
@@ -141,6 +145,7 @@ def test_prompt_caching_savings_go_negative_on_a_write_only_request():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=usage_object,
)
true_savings = _net_caching_savings_against_biller(usage_object)
@@ -156,6 +161,7 @@ def test_prompt_caching_savings_negative_when_writes_outweigh_reads():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=usage_object,
)
true_savings = _net_caching_savings_against_biller(usage_object)
@@ -172,6 +178,7 @@ def test_read_only_request_is_unchanged_by_the_write_premium():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=20000, written=0),
)
assert result.prompt_caching == pytest.approx(20000 * (input_cost - cache_read_cost))
@@ -185,12 +192,14 @@ def test_openai_style_cache_write_tokens_are_netted_out():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 5000, "cache_creation_input_tokens": 800},
)
nested_only = compute_savings_spend(
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object={
"prompt_tokens_details": {"cached_tokens": 5000, "cache_write_tokens": 800},
},
@@ -221,6 +230,7 @@ def test_model_without_a_cache_write_price_takes_no_premium():
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=5000, written=5000),
)
assert result.prompt_caching == pytest.approx(5000 * (input_cost - cache_read_cost))
@@ -244,6 +254,7 @@ def test_zero_cache_write_price_is_read_as_unpublished():
model="deepseek-chat",
custom_llm_provider="deepseek",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=0, written=10000),
)
assert result.prompt_caching == pytest.approx(0.0)
@@ -268,6 +279,7 @@ def test_zero_cache_read_price_stays_literal():
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=10000, written=0),
)
# free reads => the whole input rate is saved, not zero
@@ -293,6 +305,7 @@ def test_sub_input_cache_write_price_is_an_extra_saving():
model=model,
custom_llm_provider=None,
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=4000),
)
assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write))
@@ -306,6 +319,7 @@ def test_negative_cache_write_count_clamps_to_zero():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 1000, "cache_creation_input_tokens": -5000},
)
assert result.prompt_caching == pytest.approx(1000 * (input_cost - cache_read_cost))
@@ -316,6 +330,7 @@ def test_unknown_model_fails_open_to_zero():
model="totally-made-up-model-xyz",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 1000},
)
assert result.compression == 0.0
@@ -327,6 +342,7 @@ def test_missing_model_fails_open_to_zero():
model=None,
custom_llm_provider=None,
compression_saved_tokens=1000,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 1000},
)
assert result.compression == 0.0
@@ -338,6 +354,7 @@ def test_negative_token_counts_clamp_to_zero():
model="claude-sonnet-5",
custom_llm_provider="anthropic",
compression_saved_tokens=-500,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": -500},
)
assert result.compression == 0.0
@@ -516,6 +533,7 @@ def test_autorouter_savings_zero_without_baseline():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision=None,
usage_object=_cached_usage_object(),
)
@@ -530,6 +548,7 @@ def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch):
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision={"conversation_continuing": True},
usage_object=_cached_usage_object(),
)
@@ -543,6 +562,7 @@ def test_the_driver_is_off_until_a_baseline_is_configured():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
+ gateway_injected_cache=True,
routing_decision={"conversation_continuing": True},
usage_object=_cached_usage_object(),
)
@@ -557,6 +577,7 @@ def test_malformed_usage_object_does_not_fail_the_spend_write():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=1000,
+ gateway_injected_cache=True,
routing_decision={"conversation_continuing": True},
usage_object={"prompt_tokens": ["not", "a", "number"]},
)
@@ -573,6 +594,7 @@ def test_model_without_cache_read_pricing_yields_no_caching_savings():
model=model,
custom_llm_provider="azure",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object={"cache_read_input_tokens": 5000},
)
assert result.prompt_caching == 0.0
@@ -882,6 +904,7 @@ def test_a_baseline_recorded_on_the_decision_turns_the_driver_on():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"},
usage_object=_cached_usage_object(),
)
@@ -895,6 +918,7 @@ def test_the_configured_baseline_overrides_the_recorded_one(monkeypatch):
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision={
"conversation_continuing": True,
"savings_baseline_model": "anthropic/claude-opus-5",
@@ -923,6 +947,7 @@ def test_a_non_string_recorded_baseline_is_ignored():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": ["anthropic/claude-opus-5"]},
usage_object=_cached_usage_object(),
)
@@ -954,6 +979,7 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=20000),
model_id=deployment_id,
llm_router=lambda: router,
@@ -965,6 +991,7 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
model="claude-sonnet-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
usage_object=_caching_usage(read=1000, written=20000),
)
assert result.prompt_caching > at_public_rates.prompt_caching
@@ -995,6 +1022,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision=decision,
usage_object=_cached_usage_object(),
llm_router=lambda: router,
@@ -1003,6 +1031,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=True,
routing_decision={k: v for k, v in decision.items() if k != "savings_baseline_deployment_id"},
usage_object=_cached_usage_object(),
llm_router=lambda: router,
@@ -1021,6 +1050,7 @@ def test_recorded_savings_win_over_recomputation():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=False,
routing_decision=_routed_decision(),
usage_object=_cached_usage_object(),
recorded_autorouter_savings=0.5,
@@ -1035,6 +1065,7 @@ def test_recorded_savings_survive_an_unusable_usage_object():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=False,
routing_decision=_routed_decision(),
usage_object={"prompt_tokens": ["not", "a", "number"]},
recorded_autorouter_savings=0.25,
@@ -1047,6 +1078,7 @@ def test_a_boolean_is_not_a_recorded_savings_figure():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=False,
routing_decision=None,
usage_object=_cached_usage_object(),
recorded_autorouter_savings=True,
@@ -1063,6 +1095,7 @@ def test_rows_written_before_the_field_shipped_recompute():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
+ gateway_injected_cache=False,
routing_decision=_routed_decision(),
usage_object=_cached_usage_object(),
)
@@ -1127,3 +1160,95 @@ def test_logging_payload_never_stamps_internal_calls():
cost_breakdown=None,
)
assert internal is None
+
+
+def test_caching_savings_require_a_gateway_injected_breakpoint():
+ """The same cached usage is attributed to the gateway only when it added a breakpoint.
+
+ Client-sent cache_control and implicit provider caching (OpenAI, Gemini) produce
+ cache reads the gateway had no hand in. Those still count as caching savings the
+ customer really got, so the total is unchanged, but nothing about them is the
+ gateway's doing and the attributed figure has to stay empty.
+ """
+ input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
+ credited = compute_savings_spend(
+ model="claude-sonnet-5",
+ custom_llm_provider="anthropic",
+ compression_saved_tokens=0,
+ gateway_injected_cache=True,
+ usage_object=_caching_usage(read=8200, written=0),
+ )
+ expected = 8200 * (input_cost - cache_read_cost)
+ assert credited.prompt_caching == pytest.approx(expected)
+ assert credited.gateway_injected_caching == pytest.approx(expected)
+ unattributed = compute_savings_spend(
+ model="claude-sonnet-5",
+ custom_llm_provider="anthropic",
+ compression_saved_tokens=0,
+ gateway_injected_cache=False,
+ usage_object=_caching_usage(read=8200, written=0),
+ )
+ assert unattributed.prompt_caching == pytest.approx(expected)
+ assert unattributed.gateway_injected_caching == 0.0
+
+
+def test_unattributed_write_only_request_still_reports_its_loss_in_the_total():
+ """A write-only request really did cost more than not caching, whoever asked for it.
+
+ The attributed figure drops it because the gateway added no breakpoint, and dropping a
+ negative is why the attributed number can sit above the total rather than below it.
+ """
+ result = compute_savings_spend(
+ model="claude-sonnet-5",
+ custom_llm_provider="anthropic",
+ compression_saved_tokens=0,
+ gateway_injected_cache=False,
+ usage_object=_caching_usage(read=0, written=20000),
+ )
+ assert result.prompt_caching < 0
+ assert result.gateway_injected_caching == 0.0
+ assert result.gateway_injected_caching > result.prompt_caching
+
+
+def test_injected_request_keeps_its_negative_net():
+ """A gateway-injected write-heavy request still reports its real loss."""
+ result = compute_savings_spend(
+ model="claude-sonnet-5",
+ custom_llm_provider="anthropic",
+ compression_saved_tokens=0,
+ gateway_injected_cache=True,
+ usage_object=_caching_usage(read=0, written=20000),
+ )
+ assert result.prompt_caching < 0
+
+
+def test_attribution_does_not_touch_compression_or_autorouter_legs():
+ input_cost, _ = _anthropic_costs("claude-sonnet-5")
+ result = compute_savings_spend(
+ model="claude-sonnet-5",
+ custom_llm_provider="anthropic",
+ compression_saved_tokens=4389,
+ gateway_injected_cache=False,
+ usage_object=_caching_usage(read=8200, written=0),
+ )
+ assert result.compression == pytest.approx(4389 * input_cost)
+ assert result.prompt_caching > 0
+ assert result.gateway_injected_caching == 0.0
+
+
+def test_marks_gateway_injection_credits_only_the_deployment_that_was_injected():
+ """Every retry, failover and fallback of a request shares one metadata bucket and one
+ litellm_call_id, so the deployment is what tells those legs apart. A marker naming a
+ sibling has to read here as no injection; that is what keeps the credit on the leg
+ that earned it without any seam having to strip it. Anything that is not this row's
+ own deployment, the missing key included, is fail-closed."""
+ assert marks_gateway_injection(None, "dep-a") is False
+ assert marks_gateway_injection({}, "dep-a") is False
+ assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, "dep-a") is True
+ assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, "dep-b") is False
+ assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, None) is False
+ # injected before a deployment was chosen, so it is in the payload every leg sends
+ assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, "dep-a") is True
+ assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, None) is True
+ assert marks_gateway_injection({"litellm_call_id": "c1"}, "dep-a") is False
+ assert marks_gateway_injection({"litellm_gateway_injected_cache": True}, "dep-a") is False
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
index 8c15ead8983..4a37ae23ff4 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
@@ -2629,7 +2629,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
- "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
+ "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@@ -2725,7 +2725,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
- "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
+ "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@@ -2819,7 +2819,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
- "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
+ "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index 5b9d591ea56..87841720e43 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -3839,3 +3839,44 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully():
assert (
metadata.get("original_model_group") is None
), "original_model_group should be None when not provided"
+@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
+def test_injected_cache_breakpoints_survive_into_spend_log_metadata(bucket):
+ """The injection marker only gates savings if it reaches the spend-log row.
+
+ _get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an
+ undeclared key is dropped silently. Both buckets are covered because chat routes
+ stamp metadata while /v1/messages routes stamp litellm_metadata, and
+ record_gateway_injection writes into whichever the request carries.
+ """
+ payload = get_logging_payload(
+ kwargs={
+ "model": "claude-sonnet-5",
+ "litellm_params": {
+ bucket: {
+ "user_api_key": "test-key",
+ "litellm_gateway_injected_cache": "dep-of-this-row",
+ }
+ },
+ },
+ response_obj=litellm.ModelResponse(id="chatcmpl-injected", choices=[], usage=litellm.Usage()),
+ start_time=datetime.datetime.now(timezone.utc),
+ end_time=datetime.datetime.now(timezone.utc),
+ )
+ metadata = json.loads(payload["metadata"])
+ assert metadata["litellm_gateway_injected_cache"] == "dep-of-this-row"
+
+
+def test_passthrough_caching_carries_no_injection_marker():
+ """The negative class the gate depends on: a request whose cache_control the client
+ supplied must read as unmarked, not merely unlabelled by accident."""
+ payload = get_logging_payload(
+ kwargs={
+ "model": "claude-sonnet-5",
+ "litellm_params": {"metadata": {"user_api_key": "test-key"}},
+ },
+ response_obj=litellm.ModelResponse(id="chatcmpl-passthrough", choices=[], usage=litellm.Usage()),
+ start_time=datetime.datetime.now(timezone.utc),
+ end_time=datetime.datetime.now(timezone.utc),
+ )
+ metadata = json.loads(payload["metadata"])
+ assert metadata["litellm_gateway_injected_cache"] is None
diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
index 50ef6f29ec2..d13a2de94d0 100644
--- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
+++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
@@ -920,6 +920,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies": ["spoofed-policy"],
"policy_sources": {"spoofed-policy": "request"},
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
+ "litellm_gateway_injected_cache": "forged-deployment-id",
"_session_deployment_affinity_ttl": 999999,
"internal_call_origin": "autorouter_classifier",
"_guardrail_pipelines": [{"name": "spoofed"}],
@@ -934,6 +935,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"disable_global_guardrails": True,
"enable_prompt_caching": True,
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
+ "litellm_gateway_injected_cache": "forged-deployment-id",
"metadata": copy.deepcopy(malicious_metadata),
"litellm_metadata": copy.deepcopy(malicious_metadata),
}
@@ -952,6 +954,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
assert "disable_global_guardrails" not in updated
assert "enable_prompt_caching" not in updated
assert "routing_decision" not in updated
+ assert "litellm_gateway_injected_cache" not in updated
stripped_keys = {
"disable_global_guardrails",
@@ -966,6 +969,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"applied_policies",
"policy_sources",
"routing_decision",
+ "litellm_gateway_injected_cache",
"_session_deployment_affinity_ttl",
"internal_call_origin",
"_guardrail_pipelines",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
index 7f075e48341..ba40163a416 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
@@ -17,6 +17,7 @@ export const shortDate = (iso: string): string =>
export const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
export const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
+export const gatewayAttributedCachingOf = (m: SpendMetrics): number => m.gateway_injected_caching_savings_spend ?? 0;
export const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0;
export const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts
index 420977f8c31..8e7c1869df2 100644
--- a/ui/litellm-dashboard/src/components/UsagePage/types.ts
+++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts
@@ -12,6 +12,7 @@ export interface SpendMetrics {
compression_saved_tokens?: number;
compression_savings_spend?: number;
prompt_caching_savings_spend?: number;
+ gateway_injected_caching_savings_spend?: number;
autorouter_savings_spend?: number;
}
diff --git a/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx b/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx
index 64294e05ef3..a3fc5333057 100644
--- a/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx
+++ b/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx
@@ -6,6 +6,7 @@ import SummaryCard from "@/components/shared/SummaryCard";
import {
autorouterOf,
cachingOf,
+ gatewayAttributedCachingOf,
compressionOf,
savedTokensOf,
usd,
@@ -25,6 +26,7 @@ export const useSavingsTotals = (results: DailyData[]) =>
compression,
caching,
autorouter,
+ gatewayAttributedCaching: sumOf(gatewayAttributedCachingOf),
savedTokens: sumOf(savedTokensOf),
total: compression + caching + autorouter,
};
@@ -48,9 +50,10 @@ const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading:
/>
label.toLowerCase().replace(/\s+/g, "-");
-const SummaryCard = ({ label, value, hint, info }: SummaryCardProps) => (
+const SummaryCard = ({ label, value, hint, info, secondary }: SummaryCardProps) => (
{label}
@@ -41,8 +43,20 @@ const SummaryCard = ({ label, value, hint, info }: SummaryCardProps) => (
)}
- {value}
- {hint && {hint}
}
+
+
+
{value}
+ {hint &&
{hint}
}
+
+ {secondary && (
+
+
+
{secondary.value}
+
{secondary.label}
+
+
+ )}
+