From 2a1e3b082a27198bf5f42f811eb1ed6b74d11159 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 24 Aug 2026 15:38:34 -0400 Subject: [PATCH] feat(spend): report prompt caching savings as total and gateway-attributed `prompt_caching_savings_spend` credited every cached request, including caching a client asked for with its own `cache_control` and caching a provider does implicitly, so the number overstated what the gateway had any hand in. Gating that column in place would have fixed the overstatement by changing what the column means, leaving rows written before the change saying "all caching savings" and rows after saying "gateway-injected only" with nothing to tell them apart, and forcing a decision about rewriting history. It also breaks the cache-leakage estimate on the dashboard, whose numerator would be gated while its denominator, the cached token counts, would not, so the rate it extrapolates from would be quietly diluted. Report both instead. `prompt_caching_savings_spend` keeps meaning every net dollar caching saved, which is what a customer means by "what did caching save me", and the new `gateway_injected_caching_savings_spend` carries the subset litellm caused by injecting the breakpoints itself. Both are derived from the same marker, so this changes what is done with it rather than how it is obtained. The attributed figure is normally the smaller of the two, being a subset of the same requests, but not always: a request that writes cache it never reads has negative net savings, and excluding such a request can lift the attributed figure above the total. Also stops the marker riding into a fallback leg. The fallback rebuild spread the failed attempt's metadata forward, so a deployment that injected nothing inherited the marker and was credited anyway, which silently restored the very overstatement this separates out. --- .../migration.sql | 18 + .../litellm_proxy_extras/schema.prisma | 6 + .../anthropic_cache_control_hook.py | 73 ++- litellm/litellm_core_utils/litellm_logging.py | 20 + litellm/main.py | 2 + litellm/proxy/_types.py | 2 + litellm/proxy/db/daily_spend_bulk_upsert.py | 1 + litellm/proxy/db/db_spend_update_writer.py | 4 + .../daily_spend_update_queue.py | 4 + litellm/proxy/litellm_pre_call_utils.py | 3 + .../common_daily_activity.py | 14 + litellm/proxy/schema.prisma | 6 + litellm/proxy/spend_tracking/savings.py | 48 +- .../spend_tracking/spend_tracking_utils.py | 1 + litellm/proxy/utils.py | 1 + litellm/responses/main.py | 2 + litellm/router.py | 1 + .../anthropic_cache_control_hook.py | 7 +- .../common_daily_activity.py | 3 + schema.prisma | 6 + .../test_anthropic_cache_control_hook.py | 111 ++++ .../test_litellm_logging.py | 482 ++++++++---------- .../test_daily_spend_update_queue.py | 2 + .../proxy/db/test_daily_spend_bulk_upsert.py | 6 +- .../proxy/db/test_db_spend_update_writer.py | 104 ++++ .../test_common_daily_activity.py | 11 + .../proxy/spend_tracking/test_savings.py | 125 +++++ .../test_spend_management_endpoints.py | 6 +- .../test_spend_tracking_utils.py | 41 ++ .../proxy/test_litellm_pre_call_utils.py | 4 + .../_components/costOptimizationUtils.ts | 1 + .../src/components/UsagePage/types.ts | 1 + .../src/components/shared/SavingsTiles.tsx | 12 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 + 34 files changed, 837 insertions(+), 301 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql new file mode 100644 index 00000000000..dee5abfa269 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql @@ -0,0 +1,18 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index d9959677116..5582cf930d7 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index ef2edbf1007..62811081555 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( with_prompt_cache_breakpoint, ) from litellm.types.integrations.anthropic_cache_control_hook import ( + GATEWAY_INJECTED_CACHE_METADATA_KEY, + GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, CacheControlInjectionPoint, CacheControlMessageInjectionPoint, ) @@ -185,7 +187,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): reserved_blocks: Final = ( 1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0 ) - breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) + breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) processed_messages = self._apply_message_injections( points=applied_message_points, messages=processed_messages, @@ -194,7 +196,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) if ( openai_dialect - and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before + and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before ): non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) @@ -236,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): return provider @staticmethod - def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int: + def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int: system_blocks: Final = ( sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0 ) @@ -258,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ``max_blocks`` is reached. Injection points are honored in config order, so earlier points win when slots are scarce. """ - used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages) + used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages) limit_reached = False for point in points: @@ -454,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks - message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) - system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system) + message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) + system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system) if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks: system_already_has_cc: Final = isinstance(processed_system, list) and any( @@ -589,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): carry the mark either at the top level (Anthropic shape) or nested under ``function`` (OpenAI shape); the Anthropic chat transform accepts both. """ - if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0: + if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0: return True if tools is not None: return any( @@ -749,6 +751,52 @@ class AnthropicCacheControlHook(CustomPromptManagement): if points: non_default_params["cache_control_injection_points"] = points + @staticmethod + def record_gateway_injection( + request_kwargs: dict[str, Any], # mutable-ok: stamps the marker into the caller's live request kwargs + added: int, + deferred_points: Sequence[CacheControlInjectionPoint] | None = None, + ) -> None: + """Name the deployment whose payload the gateway, not the client, put breakpoints on. + + Spend accounting only asks whether litellm acted, so what it needs is which + deployment, not a count. Recording that is what makes the mark attempt-scoped: the + metadata bucket is one dict shared by every retry, failover and fallback of a + request, and ``litellm_call_id`` is shared with it, so anything request-scoped + written by one attempt is read by all of them and each boundary would have to + remember to strip it. The deployment is the part that actually changes when the + request moves, so a leg that injected nothing is never credited for one that did. + + It also makes a zero delta (hook re-entry) and a negative one (a prompt manager + replacing the messages) harmless, since neither rewrites an earlier mark. + + A pass that runs before a deployment is chosen, which is what the proxy does for + prompt templates, injects into the payload every leg goes on to send, so it marks + the request for all of them rather than for one. + + ``deferred_points`` are the points handed onward. A ``tool_config`` point is + placed by the provider transform rather than here, so counting only this pass's + messages would report no injection for a request litellm did cache. + + Never CREATES the bucket. The proxy seeds it on every request and is the marker's + only reader, so a request without one is a bare SDK call nothing would consume it + from. Creating it would also add a key to a dict call sites splat as ``**kwargs``, + and on the Responses API ``metadata`` is both this bucket's default name and an + explicit parameter, so the splat collides with the caller's own value. + """ + if added <= 0 and not any(p.get("location") == "tool_config" for p in deferred_points or ()): + return + from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs + + bucket: Final = request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) + if isinstance(bucket, dict): + model_info: Final = request_kwargs.get("model_info") + bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = ( + model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT) + if isinstance(model_info, dict) + else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT + ) + @staticmethod def maybe_inject_cache_control( messages: list[dict], @@ -798,17 +846,18 @@ class AnthropicCacheControlHook(CustomPromptManagement): openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") ) - breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) + breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( messages=messages, system=system, injection_points=injection_points, openai_dialect=openai_dialect, ) - if ( - openai_dialect - and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before - ): + breakpoints_added: Final = ( + AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before + ) + AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added, remaining) + if openai_dialect and breakpoints_added > 0: kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) if remaining: kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 626af7530a4..b79246a218b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -864,7 +864,10 @@ class Logging(LiteLLMLoggingBaseClass): prompt_management_logger: CustomLogger | None = None, prompt_label: str | None = None, prompt_version: int | None = None, + request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs ) -> tuple[str, list[AllMessageValues], dict]: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management( model=model, non_default_params=non_default_params, @@ -874,6 +877,7 @@ class Logging(LiteLLMLoggingBaseClass): ) if custom_logger: + breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages) ( model, messages, @@ -889,6 +893,12 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label=prompt_label, prompt_version=prompt_version, ) + if request_kwargs is not None: + AnthropicCacheControlHook.record_gateway_injection( + request_kwargs, + AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + non_default_params.get("cache_control_injection_points"), + ) self.messages = messages return model, messages, non_default_params @@ -904,7 +914,10 @@ class Logging(LiteLLMLoggingBaseClass): tools: list[dict] | None = None, prompt_label: str | None = None, prompt_version: int | None = None, + request_kwargs: dict[str, object] | None = None, # mutable-ok: marker stamped into live request kwargs ) -> tuple[str, list[AllMessageValues], dict]: + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + custom_logger: Final = prompt_management_logger or self.get_custom_logger_for_prompt_management( model=model, tools=tools, @@ -915,6 +928,7 @@ class Logging(LiteLLMLoggingBaseClass): ) if custom_logger: + breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages) ( model, messages, @@ -932,6 +946,12 @@ class Logging(LiteLLMLoggingBaseClass): prompt_label=prompt_label, prompt_version=prompt_version, ) + if request_kwargs is not None: + AnthropicCacheControlHook.record_gateway_injection( + request_kwargs, + AnthropicCacheControlHook.count_request_cache_breakpoints(messages) - breakpoints_before, + non_default_params.get("cache_control_injection_points"), + ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/main.py b/litellm/main.py index 98f92e50599..d3a5bcbcab6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -531,6 +531,7 @@ async def acompletion( tools=tools, prompt_label=kwargs.get("prompt_label", None), prompt_version=kwargs.get("prompt_version", None), + request_kwargs=kwargs, ) ######################################################### # if the chat completion logging hook removed all tools, @@ -5245,6 +5246,7 @@ def completion( prompt_variables=prompt_variables, prompt_label=kwargs.get("prompt_label", None), prompt_version=kwargs.get("prompt_version", None), + request_kwargs=kwargs, ) ### LITELLM SYSTEM PROMPT ### diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 61acac3ff74..f210f6349ab 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3551,6 +3551,7 @@ class SpendLogsMetadata(TypedDict): cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.) compression_savings: CompressionSavingsMetadata | None autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed + litellm_gateway_injected_cache: ReadOnly[str | None] class SpendLogsPayload(TypedDict): @@ -4831,6 +4832,7 @@ class BaseDailySpendTransaction(TypedDict): # cost-savings metrics (dollars, priced per request before aggregation) compression_savings_spend: float prompt_caching_savings_spend: float + gateway_injected_caching_savings_spend: float # writable-ok: the rollup queue accumulates into this key in place, as it does for every sibling spend field # Not required: rows queued by a pod running the previous release, or replayed from # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a # missing value to zero, so requiring it here would describe a shape the aggregation diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index 55d325177c6..a143643577e 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -62,6 +62,7 @@ _SPEND_COLUMNS: Final = ( "spend", "compression_savings_spend", "prompt_caching_savings_spend", + "gateway_injected_caching_savings_spend", "autorouter_savings_spend", ) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 0c8c9a853ec..3f2777ff1f3 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -62,6 +62,7 @@ from litellm.proxy.spend_tracking.savings import ( compute_savings_spend, extract_cache_creation_tokens, extract_cache_read_tokens, + marks_gateway_injection, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable @@ -315,6 +316,7 @@ class DBSpendUpdateWriter: model=payload.get("model"), custom_llm_provider=payload.get("custom_llm_provider"), compression_saved_tokens=0, + gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")), routing_decision=metadata.get("routing_decision"), usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None, model_id=payload.get("model_id"), @@ -1879,6 +1881,7 @@ class DBSpendUpdateWriter: model=payload.get("model", None), custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, + gateway_injected_cache=marks_gateway_injection(_metadata, payload.get("model_id")), routing_decision=_metadata.get("routing_decision"), model_id=payload.get("model_id"), llm_router=_get_llm_router, @@ -1911,6 +1914,7 @@ class DBSpendUpdateWriter: compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, + gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching, autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter, ) return daily_transaction diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 6a97d010b35..70a529900b2 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("prompt_caching_savings_spend", 0) or 0 ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["gateway_injected_caching_savings_spend"] = ( + payload.get("gateway_injected_caching_savings_spend", 0) or 0 + ) + daily_transaction.get("gateway_injected_caching_savings_spend", 0) + daily_transaction["autorouter_savings_spend"] = ( payload.get("autorouter_savings_spend", 0) or 0 ) + daily_transaction.get("autorouter_savings_spend", 0) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 064b53e07b7..26a9bc7f1c6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -50,6 +50,7 @@ from litellm.proxy.common_utils.callback_utils import ( strip_callback_config, ) from litellm.proxy.common_utils.http_parsing_utils import _safe_get_request_headers +from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE: Final = frozenset(v.value.lower() for v in SpecialHeaders._member_map_.values()) @@ -220,6 +221,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "policy_sources", "guardrail_scan_ids", "routing_decision", + GATEWAY_INJECTED_CACHE_METADATA_KEY, "pillar_response_headers", "_guardrail_pipelines", "_pipeline_managed_guardrails", @@ -274,6 +276,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "policy_sources", "guardrail_scan_ids", "routing_decision", + GATEWAY_INJECTED_CACHE_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index d3968bf323b..91cd80b3c81 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -94,6 +94,9 @@ class DailySpendRecord(Protocol): @property def prompt_caching_savings_spend(self) -> float: ... + @property + def gateway_injected_caching_savings_spend(self) -> float: ... + @property def autorouter_savings_spend(self) -> float: ... @@ -137,6 +140,7 @@ class _GroupingSetsRow(SimpleNamespace): compression_saved_tokens: int | None compression_savings_spend: float | None prompt_caching_savings_spend: float | None + gateway_injected_caching_savings_spend: float | None autorouter_savings_spend: float | None api_requests: int | None successful_requests: int | None @@ -189,6 +193,9 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 + existing_metrics.gateway_injected_caching_savings_spend += ( # rebind-ok: this accumulator mutates its target in place for every metric on the row + record.gateway_injected_caching_savings_spend or 0 + ) existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 @@ -721,6 +728,7 @@ def _build_aggregated_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend, SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, @@ -799,6 +807,7 @@ def _build_entity_rollup_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(gateway_injected_caching_savings_spend)::float AS gateway_injected_caching_savings_spend, SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, @@ -934,6 +943,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: compression_saved_tokens=record.compression_saved_tokens or 0, compression_savings_spend=record.compression_savings_spend or 0, prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, + gateway_injected_caching_savings_spend=record.gateway_injected_caching_savings_spend or 0, autorouter_savings_spend=record.autorouter_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, @@ -1200,6 +1210,7 @@ async def get_daily_activity( total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, total_compression_savings_spend=metadata_metrics.compression_savings_spend, total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, + total_gateway_injected_caching_savings_spend=metadata_metrics.gateway_injected_caching_savings_spend, total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division @@ -1372,6 +1383,9 @@ async def get_daily_activity_aggregated( total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, total_compression_savings_spend=aggregated["totals"].compression_savings_spend, total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, + total_gateway_injected_caching_savings_spend=aggregated[ + "totals" + ].gateway_injected_caching_savings_spend, total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, page=1, total_pages=1, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index d9959677116..5582cf930d7 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index b0f1546e15e..7f412da32d1 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -15,6 +15,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token +from litellm.types.integrations.anthropic_cache_control_hook import ( + GATEWAY_INJECTED_CACHE_METADATA_KEY, + GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, +) if TYPE_CHECKING: from litellm.router import Router @@ -25,6 +29,7 @@ class SavingsSpend(NamedTuple): compression: float prompt_caching: float autorouter: float = 0.0 + gateway_injected_caching: float = 0.0 def _input_cache_read_and_write_cost(info: ModelInfo | None) -> tuple[float, float, float]: @@ -391,6 +396,28 @@ def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | return None +def marks_gateway_injection(metadata: Mapping[str, object] | None, model_id: str | None) -> bool: + """Whether the gateway put cache breakpoints on the payload THIS row was billed for. + + ``AnthropicCacheControlHook.record_gateway_injection`` stamps the deployment it + injected for, and a row carries the deployment it was billed for, so the two agree + only on the leg that was actually injected. Every retry, failover and fallback of a + request shares one metadata bucket and one ``litellm_call_id``, so the deployment is + what tells those legs apart, and a marker left by a sibling reads here as no injection + without anyone having to strip it. An injection that ran before any deployment was + chosen is in the payload every leg sends, so it is marked for all of them and credits + each. Absent on requests the gateway never acted on + (client-supplied ``cache_control``, implicit provider caching) and on rows written + before the marker shipped; all of it is the fail-closed direction. + """ + if not metadata: + return False + injected_deployment: Final = metadata.get(GATEWAY_INJECTED_CACHE_METADATA_KEY) + if not isinstance(injected_deployment, str): + return False + return injected_deployment in (GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT, model_id) + + def extract_cache_read_tokens(usage_object: Mapping[str, object] | None) -> int: """Cache-read tokens from a logged usage object, whatever shape recorded them. @@ -533,6 +560,7 @@ def compute_savings_spend( model: str | None, custom_llm_provider: str | None, compression_saved_tokens: int, + gateway_injected_cache: bool, routing_decision: Mapping[str, object] | None = None, usage_object: Mapping[str, object] | None = None, model_id: str | None = None, @@ -565,7 +593,23 @@ def compute_savings_spend( A request that only writes cache and gets no hits therefore reports negative savings, which is accurate: it really did cost more than the uncached call would have. The daily rollup increments arithmetically, so those rows offset positive ones in the - same bucket. Auto-router savings compare the + same bucket. + + Caching is reported twice. ``prompt_caching`` is every net dollar caching saved, + whoever caused it, which is what a customer means by "what did caching save me". + ``gateway_injected_caching`` is the subset the gateway can claim credit for, carrying + a value only when ``gateway_injected_cache`` is set, i.e. litellm itself added the + ``cache_control`` breakpoints (configured injection points or the auto prompt-caching + flag). A client that sent its own breakpoints, and a provider that + caches implicitly (OpenAI, Gemini), produce the same usage shape with no gateway + action, so they count toward the total and not toward the attributed figure. + + Reporting both rather than gating the one column keeps the customer-facing number + stable across the change and leaves attribution a separate question. The attributed + figure is normally the smaller of the two, being a subset of the same requests, but + not always: a request that only writes cache and never reads it has negative net + savings, and dropping such a request from the attributed figure can lift it above + the total. Auto-router savings compare the served ``model`` against the counterfactual baseline the router recorded on its ``routing_decision``, and are zero unless the two differ. That record also says whether the conversation was already underway, which is what tells @@ -602,6 +646,7 @@ def compute_savings_spend( read_discount: Final = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost) prompt_caching: Final = read_discount - write_premium + gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0 # The figure the logging path recorded wins, before the usage gate on purpose: a row # whose usage no longer parses still carries the number computed when it did. @@ -623,4 +668,5 @@ def compute_savings_spend( compression=compression, prompt_caching=prompt_caching, autorouter=0.0 if autorouter is None else autorouter, + gateway_injected_caching=gateway_injected_caching, ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 38da38ead2b..9858999111c 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -136,6 +136,7 @@ def _get_spend_logs_metadata( cost_breakdown=None, compression_savings=None, autorouter_savings=autorouter_savings, + litellm_gateway_injected_cache=None, litellm_call_id=litellm_call_id, ) verbose_proxy_logger.debug( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8cbf5b685fd..78a7e9ae18f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1434,6 +1434,7 @@ class ProxyLogging: prompt_variables=data.pop("prompt_variables", None) or {}, prompt_label=data.pop("prompt_label", None) or {}, prompt_version=data.pop("prompt_version", None) or {}, + request_kwargs=data, ) data.update(optional_params) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8a682c701e0..4ec37eba060 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -541,6 +541,7 @@ async def aresponses( prompt_variables=prompt_variables, prompt_label=kwargs.get("prompt_label", None), prompt_version=kwargs.get("prompt_version", None), + request_kwargs=kwargs, ) input = cast( str | ResponseInputParam, @@ -664,6 +665,7 @@ def _apply_prompt_management_to_responses_call( prompt_variables=prompt_variables, prompt_label=kwargs.get("prompt_label", None), prompt_version=kwargs.get("prompt_version", None), + request_kwargs=kwargs, ) input = cast( str | ResponseInputParam, diff --git a/litellm/router.py b/litellm/router.py index 2dc79670c07..ab850237972 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3959,6 +3959,7 @@ class Router: prompt_id=prompt_id, prompt_variables=prompt_variables, prompt_label=prompt_label, + request_kwargs=kwargs, ) # Filter out prompt management specific parameters from data before merging diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 3ab0c02f28d..ef414f22c3b 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -1,9 +1,14 @@ -from typing import Literal +from typing import Final, Literal from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.types.llms.openai import ChatCompletionCachedContent +GATEWAY_INJECTED_CACHE_METADATA_KEY: Final = "litellm_gateway_injected_cache" +# No deployment had been chosen when the injection happened, so it is in the payload +# every leg of the request sends. Never a real deployment id. +GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT: Final = "" + class CacheControlMessageInjectionPoint(TypedDict): """Type for message-level injection points.""" diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 16d08b33150..101405abf50 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -26,6 +26,7 @@ class SpendMetrics(BaseModel): compression_saved_tokens: int = Field(default=0) compression_savings_spend: float = Field(default=0.0) prompt_caching_savings_spend: float = Field(default=0.0) + gateway_injected_caching_savings_spend: float = Field(default=0.0) autorouter_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) @@ -88,6 +89,7 @@ class DailySpendMetadata(BaseModel): total_compression_saved_tokens: int = Field(default=0) total_compression_savings_spend: float = Field(default=0.0) total_prompt_caching_savings_spend: float = Field(default=0.0) + total_gateway_injected_caching_savings_spend: float = Field(default=0.0) total_autorouter_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) @@ -115,6 +117,7 @@ class LiteLLM_DailyUserSpend(BaseModel): compression_saved_tokens: int = 0 compression_savings_spend: float = 0.0 prompt_caching_savings_spend: float = 0.0 + gateway_injected_caching_savings_spend: float = 0.0 autorouter_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 diff --git a/schema.prisma b/schema.prisma index d9959677116..5582cf930d7 100644 --- a/schema.prisma +++ b/schema.prisma @@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) @@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index cf3318b32fe..27f87225c98 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -2796,3 +2796,114 @@ class TestPromptCacheBreakpointCapability: def test_unlisted_model_falls_back_to_the_version_rule(self, model, expected): assert model not in litellm.model_cost assert supports_openai_prompt_cache_breakpoint(model) is expected + + +class TestRecordGatewayInjection: + """The injection marker spend accounting gates prompt-caching savings on.""" + + KEY = "litellm_gateway_injected_cache" + DEPLOYMENT = "dep-abc" + + def test_records_only_an_actual_injection(self): + """A zero delta is hook re-entry and a negative one is a prompt manager replacing + the messages; neither is litellm adding a breakpoint.""" + kwargs: dict = {"metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 0) + AnthropicCacheControlHook.record_gateway_injection(kwargs, -3) + assert kwargs["metadata"] == {} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 2) + assert kwargs["metadata"][self.KEY] == self.DEPLOYMENT + + def test_records_a_point_the_provider_transform_will_place(self): + """A tool_config point is placed on the Bedrock tools by the provider transform, + after this pass counted its messages, so a delta of zero still means litellm + cached something and the request must not read as untouched.""" + kwargs: dict = {"metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 0, [{"location": "tool_config"}]) + assert kwargs["metadata"][self.KEY] == self.DEPLOYMENT + + @pytest.mark.parametrize("kwargs", [{}, {"metadata": None}, {"metadata": "not-a-dict"}]) + def test_never_introduces_a_metadata_key(self, kwargs): + """Stamping must not add a key to a dict the caller splats as ``**kwargs``. + + ``aresponses`` takes ``metadata`` as an explicit parameter and forwards the rest + of the request as ``**kwargs``, so a bucket created here arrives twice and the + call dies with "got multiple values for keyword argument 'metadata'". Only the + proxy reads this marker and it always seeds the bucket first, so a request + without one has nothing to record. + """ + before = dict(kwargs) + AnthropicCacheControlHook.record_gateway_injection(kwargs, 3) + assert kwargs == before + + def test_a_later_pass_cannot_unset_an_earlier_injection(self): + kwargs: dict = {"litellm_metadata": {"user_api_key": "k"}, "model_info": {"id": self.DEPLOYMENT}} + AnthropicCacheControlHook.record_gateway_injection(kwargs, 2) + AnthropicCacheControlHook.record_gateway_injection(kwargs, 0) + assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + + def test_v1_messages_auto_injection_stamps_the_marker(self, monkeypatch): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + result_msgs, result_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": "latest turn"}], + "a long system prompt", + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + + def test_v1_messages_stand_down_leaves_no_marker(self, monkeypatch): + """Client-supplied cache_control means the gateway did nothing to credit.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + kwargs: dict = {"litellm_metadata": {}} + AnthropicCacheControlHook.maybe_inject_cache_control( + [ + { + "role": "system", + "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "latest turn"}, + ], + None, + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert self.KEY not in kwargs["litellm_metadata"] + + def test_v1_messages_reentry_keeps_the_marker(self, monkeypatch): + """A second pass over already-injected messages computes a zero delta, which must + leave the first pass's mark standing rather than reading as no injection.""" + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + kwargs: dict = {"litellm_metadata": {}, "model_info": {"id": self.DEPLOYMENT}} + messages = [{"role": "user", "content": "latest turn"}] + first_msgs, first_sys = AnthropicCacheControlHook.maybe_inject_cache_control( + messages, "a long system prompt", kwargs, model="claude-sonnet-4-5", custom_llm_provider="anthropic" + ) + AnthropicCacheControlHook.maybe_inject_cache_control( + first_msgs, first_sys, kwargs, model="claude-sonnet-4-5", custom_llm_provider="anthropic" + ) + assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT + + def test_configured_points_skipping_a_marked_target_record_nothing(self): + """Configured injection stands down on client breakpoints, so no marker lands.""" + kwargs: dict = { + "litellm_metadata": {}, + "cache_control_injection_points": [{"location": "message", "role": "system", "index": None}], + } + AnthropicCacheControlHook.maybe_inject_cache_control( + [ + { + "role": "system", + "content": [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "hi"}, + ], + None, + kwargs, + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + assert self.KEY not in kwargs["litellm_metadata"] diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 29b283ec009..daa5dba3d7b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -273,9 +273,7 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): assert cost is not None, "Cost should not be None" expected_cost = (10 * custom_input_cost) + (5 * custom_output_cost) - assert cost == pytest.approx( - expected_cost - ), f"Expected {expected_cost}, got {cost}" + assert cost == pytest.approx(expected_cost), f"Expected {expected_cost}, got {cost}" finally: litellm.model_cost.pop(custom_model_id, None) @@ -872,13 +870,8 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch): # Regression check: we expect a distinct DataDogLogger, not the LLM Obs logger assert type(datadog_logger) is DataDogLogger - assert any( - isinstance(cb, DataDogLLMObsLogger) - for cb in logging_module._in_memory_loggers - ) - assert any( - type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers - ) + assert any(isinstance(cb, DataDogLLMObsLogger) for cb in logging_module._in_memory_loggers) + assert any(type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers) finally: logging_module._in_memory_loggers.clear() @@ -889,9 +882,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): # Required env vars for Logfire integration monkeypatch.setenv("LOGFIRE_TOKEN", "test-token") - monkeypatch.setenv( - "LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev" - ) # no trailing slash on purpose + monkeypatch.setenv("LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev") # no trailing slash on purpose # Import after env vars are set (important if module-level caching exists) from litellm.integrations.opentelemetry import OpenTelemetry # logger class @@ -910,9 +901,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): # Sanity: we got the right logger type and it is cached assert type(logger) is OpenTelemetry - assert any( - type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers - ) + assert any(type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers) # Core regression check: base URL env var should influence the exporter endpoint. # @@ -923,9 +912,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): or getattr(logger, "config", None) or getattr(logger, "_otel_config", None) ) - assert ( - cfg is not None - ), "Expected OpenTelemetry logger to keep an otel config on the instance" + assert cfg is not None, "Expected OpenTelemetry logger to keep an otel config on the instance" endpoint = getattr(cfg, "endpoint", None) or getattr(cfg, "otlp_endpoint", None) assert endpoint is not None, "Expected otel config to expose the OTLP endpoint" @@ -1083,9 +1070,7 @@ async def test_logging_non_streaming_request(): # Use the filtered call for assertions call_args = calls_with_expected_input[0] - standard_logging_object = call_args.kwargs["kwargs"][ - "standard_logging_object" - ] + standard_logging_object = call_args.kwargs["kwargs"]["standard_logging_object"] assert standard_logging_object["stream"] is not True finally: # Restore original callbacks to ensure test isolation @@ -1103,18 +1088,14 @@ async def test_logging_non_streaming_request(): "agenerate_content_stream", ], ) -def test_success_handler_skips_sync_callbacks_for_async_requests( - logging_obj, async_flag -): +def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, async_flag): """Ensure sync success callbacks are skipped when async call type flags are set.""" from litellm.integrations.custom_logger import CustomLogger class DummyLogger(CustomLogger): pass - logging_obj.stream = ( - False # simulate non-streaming request where sync callbacks would normally run - ) + logging_obj.stream = False # simulate non-streaming request where sync callbacks would normally run logging_obj.model_call_details["litellm_params"] = {async_flag: True} logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] @@ -1190,21 +1171,11 @@ def test_success_handler_runs_sync_callbacks_for_sync_requests(logging_obj, call def test_is_sync_litellm_request(): assert LitellmLogging._is_sync_litellm_request({}) is True assert LitellmLogging._is_sync_litellm_request({"acompletion": True}) is False - assert ( - LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True}) - is False - ) - assert ( - LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False - ) + assert LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True}) is False + assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False assert LitellmLogging._is_sync_litellm_request({"agenerate_content": True}) is False - assert ( - LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True}) - is False - ) - assert ( - LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True - ) + assert LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True}) is False + assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True def test_get_litellm_params_propagates_allm_passthrough_route(): @@ -1251,9 +1222,7 @@ async def test_dispatch_success_handlers_invokes_callbacks_once_for_final_stream logging_obj.model_call_details["litellm_params"] = {"acompletion": True} with ( - patch.object( - mock_callback, "async_log_success_event", new_callable=AsyncMock - ) as mock_async_log, + patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log, patch.object(mock_callback, "log_success_event") as mock_sync_log, patch.object( logging_obj, @@ -1314,9 +1283,7 @@ async def test_dispatch_success_handlers_sync_path_invokes_callback_once_for_fin with ( patch.object(mock_callback, "log_success_event") as mock_sync_log, - patch.object( - mock_callback, "async_log_success_event", new_callable=AsyncMock - ) as mock_async_log, + patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log, patch.object( logging_obj, "_success_handler_helper_fn", @@ -1358,20 +1325,14 @@ async def test_dispatch_prefer_async_handlers_runs_legacy_callbacks( logging_obj.model_call_details["litellm_params"] = {} with ( - patch.object( - logging_obj, "async_success_handler", new_callable=AsyncMock - ) as mock_async, - patch.object( - logging_obj, "success_handler", new_callable=MagicMock - ) as mock_sync, + patch.object(logging_obj, "async_success_handler", new_callable=AsyncMock) as mock_async, + patch.object(logging_obj, "success_handler", new_callable=MagicMock) as mock_sync, patch.object( logging_obj, "_should_run_sync_callbacks_for_async_calls", return_value=True, ), - patch( - "litellm.litellm_core_utils.litellm_logging.executor.submit" - ) as mock_submit, + patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit, ): await logging_obj.dispatch_success_handlers( result=result, @@ -1405,9 +1366,7 @@ async def test_dispatch_success_handlers_invokes_async_callback_for_pass_through try: with ( - patch.object( - mock_callback, "async_log_success_event", new_callable=AsyncMock - ) as mock_async_log, + patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log, patch.object(mock_callback, "log_success_event") as mock_sync_log, ): await logging_obj.dispatch_success_handlers(result={"id": "pt-1"}) @@ -1434,20 +1393,14 @@ async def test_dispatch_failure_handlers_prefer_async_does_not_submit_sync_handl logging_obj.model_call_details["litellm_params"] = {} with ( - patch.object( - logging_obj, "async_failure_handler", new_callable=AsyncMock - ) as mock_async, - patch.object( - logging_obj, "failure_handler", new_callable=MagicMock - ) as mock_sync, + patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async, + patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync, patch.object( logging_obj, "_should_run_sync_failure_callbacks_for_async_calls", return_value=False, ), - patch( - "litellm.litellm_core_utils.litellm_logging.executor.submit" - ) as mock_submit, + patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit, ): await logging_obj.dispatch_failure_handlers( exception, @@ -1530,12 +1483,8 @@ async def test_dispatch_failure_handlers_submits_sync_handler_for_failure_only_c patch.object(litellm, "success_callback", []), patch.object(litellm, "failure_callback", [_sync_failure_callback]), patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock), - patch.object( - logging_obj, "failure_handler", new_callable=MagicMock - ) as mock_sync, - patch( - "litellm.litellm_core_utils.litellm_logging.executor.submit" - ) as mock_submit, + patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync, + patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit, ): await logging_obj.dispatch_failure_handlers( exception, @@ -1562,15 +1511,9 @@ async def test_dispatch_failure_handlers_sync_sdk_shortcut_runs_sync_handler_inl logging_obj.model_call_details["litellm_params"] = {} with ( - patch.object( - logging_obj, "async_failure_handler", new_callable=AsyncMock - ) as mock_async, - patch.object( - logging_obj, "failure_handler", new_callable=MagicMock - ) as mock_sync, - patch( - "litellm.litellm_core_utils.litellm_logging.executor.submit" - ) as mock_submit, + patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async, + patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync, + patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit, ): await logging_obj.dispatch_failure_handlers( exception, @@ -1617,14 +1560,10 @@ def test_success_handler_skips_guardrail_logging_hook_when_disabled(logging_obj) event_hook=GuardrailEventHooks.logging_only, ) guardrail.should_run_guardrail = MagicMock(return_value=False) - guardrail.logging_hook = MagicMock( - return_value=(logging_obj.model_call_details, model_response) - ) + guardrail.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response)) dummy_logger = DummyLogger() - dummy_logger.logging_hook = MagicMock( - return_value=(logging_obj.model_call_details, model_response) - ) + dummy_logger.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response)) with patch.object( logging_obj, @@ -1758,11 +1697,7 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): # Test case 2: Tags in litellm_metadata only tags = StandardLoggingPayloadSetup._get_request_tags( - litellm_params={ - "litellm_metadata": { - "tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"] - } - }, + litellm_params={"litellm_metadata": {"tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]}}, proxy_server_request={}, ) assert "litellm-metadata-tag-1" in tags @@ -1867,15 +1802,9 @@ def test_get_request_tags_does_not_mutate_original_tags(): user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")]) user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")]) - assert ( - user_agent_count_1 == 2 - ), f"Expected 2 User-Agent tags, got {user_agent_count_1}" - assert ( - user_agent_count_2 == 2 - ), f"Expected 2 User-Agent tags, got {user_agent_count_2}" - assert ( - user_agent_count_3 == 2 - ), f"Expected 2 User-Agent tags, got {user_agent_count_3}" + assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}" + assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}" + assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}" # Verify all returned lists are independent (different objects) assert tags1 is not tags2 @@ -1908,9 +1837,7 @@ def test_get_extra_header_tags(): # Test case 3: Extra headers configured but request has no headers dict litellm.extra_spend_tag_headers = ["x-custom", "x-tenant"] - result = StandardLoggingPayloadSetup._get_extra_header_tags( - proxy_server_request={"headers": "not-a-dict"} - ) + result = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": "not-a-dict"}) assert result is None # Test case 4: Extra headers configured but none match request headers @@ -2211,9 +2138,7 @@ def test_get_masked_values(): "presidio_anonymizer_api_base": None, "vertex_credentials": "{sensitive_api_key}", } - masked_values = _get_masked_values( - sensitive_object, unmasked_length=4, number_of_asterisks=4 - ) + masked_values = _get_masked_values(sensitive_object, unmasked_length=4, number_of_asterisks=4) assert masked_values["presidio_anonymizer_api_base"] is None assert masked_values["vertex_credentials"] == "{s****y}" @@ -2238,9 +2163,7 @@ async def test_e2e_generate_cold_storage_object_key_successful(): patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key, ): # Mock the S3 object key generation to return a predictable result - mock_get_s3_key.return_value = ( - "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" - ) + mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" # Call the function result = StandardLoggingPayloadSetup._generate_cold_storage_object_key( @@ -2281,16 +2204,12 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path() with ( patch("litellm.cold_storage_custom_logger", "s3_v2"), - patch( - "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name" - ) as mock_get_logger, + patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger, patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key, ): # Setup mocks mock_get_logger.return_value = mock_custom_logger - mock_get_s3_key.return_value = ( - "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" - ) + mock_get_s3_key.return_value = "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" # Call the function result = StandardLoggingPayloadSetup._generate_cold_storage_object_key( @@ -2309,9 +2228,7 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path() ) # Verify the result - assert ( - result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" - ) + assert result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" @pytest.mark.asyncio @@ -2334,16 +2251,12 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path(): with ( patch("litellm.cold_storage_custom_logger", "s3_v2"), - patch( - "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name" - ) as mock_get_logger, + patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger, patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key, ): # Setup mocks mock_get_logger.return_value = mock_custom_logger - mock_get_s3_key.return_value = ( - "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" - ) + mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json" # Call the function result = StandardLoggingPayloadSetup._generate_cold_storage_object_key( @@ -2459,9 +2372,7 @@ def test_get_usage_as_dict(): assert result == {"prompt_tokens": 20, "completion_tokens": 30} # Test case 5: response_obj with no usage key returns empty - result = StandardLoggingPayloadSetup.get_usage_as_dict( - response_obj={"id": "resp-1", "choices": []} - ) + result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj={"id": "resp-1", "choices": []}) assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} @@ -2474,26 +2385,20 @@ def test_append_system_prompt_messages(): # Test case 1: system in kwargs with existing messages kwargs = {"system": "You are a helpful assistant"} messages = [{"role": "user", "content": "Hello"}] - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=messages - ) + result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages) assert len(result) == 2 assert result[0] == {"role": "system", "content": "You are a helpful assistant"} assert result[1] == {"role": "user", "content": "Hello"} # Test case 2: system in kwargs with None messages kwargs = {"system": "You are a helpful assistant"} - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=None - ) + result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=None) assert len(result) == 1 assert result[0] == {"role": "system", "content": "You are a helpful assistant"} # Test case 3: system in kwargs with empty messages list kwargs = {"system": "You are a helpful assistant"} - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=[] - ) + result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=[]) assert len(result) == 1 assert result[0] == {"role": "system", "content": "You are a helpful assistant"} @@ -2503,24 +2408,18 @@ def test_append_system_prompt_messages(): {"role": "system", "content": "You are a helpful assistant"}, {"role": "user", "content": "Hello"}, ] - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=messages - ) + result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages) assert len(result) == 2 assert result[0] == {"role": "system", "content": "You are a helpful assistant"} # Test case 5: no system in kwargs returns messages unchanged kwargs = {} messages = [{"role": "user", "content": "Hello"}] - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=kwargs, messages=messages - ) + result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages) assert result == messages # Test case 6: None kwargs returns messages unchanged - result = StandardLoggingPayloadSetup.append_system_prompt_messages( - kwargs=None, messages=messages - ) + result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=None, messages=messages) assert result == messages @@ -2581,12 +2480,11 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu # Verify that standard_logging_object was set assert "standard_logging_object" in logging_obj.model_call_details, ( - "standard_logging_object should be set for pass-through endpoints " - "even when complete_streaming_response is None" + "standard_logging_object should be set for pass-through endpoints even when complete_streaming_response is None" + ) + assert logging_obj.model_call_details["standard_logging_object"] is not None, ( + "standard_logging_object should not be None for pass-through endpoints" ) - assert ( - logging_obj.model_call_details["standard_logging_object"] is not None - ), "standard_logging_object should not be None for pass-through endpoints" # Verify that async_complete_streaming_response was set to prevent re-processing # This is consistent with the existing code pattern for regular streaming @@ -2594,15 +2492,13 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu "async_complete_streaming_response should be set to prevent re-processing, " "consistent with the existing code pattern" ) - assert ( - logging_obj.model_call_details["async_complete_streaming_response"] is result - ), "async_complete_streaming_response should be set to the result" + assert logging_obj.model_call_details["async_complete_streaming_response"] is result, ( + "async_complete_streaming_response should be set to the result" + ) # Verify that response_cost is set to None (cost calculation not possible for pass-through) # This is consistent with the error handling in the non-pass-through code path - assert ( - "response_cost" in logging_obj.model_call_details - ), "response_cost should be set for pass-through endpoints" + assert "response_cost" in logging_obj.model_call_details, "response_cost should be set for pass-through endpoints" assert logging_obj.model_call_details["response_cost"] is None, ( "response_cost should be None for pass-through endpoints since " "StandardPassThroughResponseObject doesn't have standard usage info" @@ -2661,14 +2557,10 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp # Verify first call set the values assert "standard_logging_object" in logging_obj.model_call_details assert "async_complete_streaming_response" in logging_obj.model_call_details - first_standard_logging_object = logging_obj.model_call_details[ - "standard_logging_object" - ] + first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"] # Second call - should return early due to async_complete_streaming_response guard - with patch.object( - logging_obj, "get_combined_callback_list", return_value=[] - ) as mock_callbacks: + with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks: await logging_obj.async_success_handler( result=result, start_time=start_time, @@ -2679,10 +2571,9 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp mock_callbacks.assert_not_called() # Verify standard_logging_object wasn't modified by second call - assert ( - logging_obj.model_call_details["standard_logging_object"] - is first_standard_logging_object - ), "standard_logging_object should not be modified on re-processing" + assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, ( + "standard_logging_object should not be modified on re-processing" + ) @pytest.mark.asyncio @@ -2721,9 +2612,7 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_ } # Create a pass-through response object (simulating unparseable streaming response) - result = StandardPassThroughResponseObject( - response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]' - ) + result = StandardPassThroughResponseObject(response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]') start_time = datetime.now() end_time = datetime.now() @@ -2743,9 +2632,9 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_ "standard_logging_object should be set for streaming pass-through endpoints " "even when the response cannot be parsed into a ModelResponse" ) - assert ( - logging_obj.model_call_details["standard_logging_object"] is not None - ), "standard_logging_object should not be None for streaming pass-through endpoints" + assert logging_obj.model_call_details["standard_logging_object"] is not None, ( + "standard_logging_object should not be None for streaming pass-through endpoints" + ) def test_get_error_information_error_code_priority(): @@ -2787,30 +2676,22 @@ def test_get_error_information_error_code_priority(): self.message = message super().__init__(message) - both_exception = BothAttributesException( - code="400", status_code=500, message="Bad Request" - ) + both_exception = BothAttributesException(code="400", status_code=500, message="Bad Request") result = StandardLoggingPayloadSetup.get_error_information(both_exception) assert result["error_code"] == "400" # Should prefer 'code' over 'status_code' # Test case 4: Exception with 'code' as empty string - should fall back to 'status_code' - empty_code_exception = BothAttributesException( - code="", status_code=404, message="Not Found" - ) + empty_code_exception = BothAttributesException(code="", status_code=404, message="Not Found") result = StandardLoggingPayloadSetup.get_error_information(empty_code_exception) assert result["error_code"] == "404" # Should fall back to status_code # Test case 5: Exception with 'code' as "None" string - should fall back to 'status_code' - none_string_exception = BothAttributesException( - code="None", status_code=503, message="Service Unavailable" - ) + none_string_exception = BothAttributesException(code="None", status_code=503, message="Service Unavailable") result = StandardLoggingPayloadSetup.get_error_information(none_string_exception) assert result["error_code"] == "503" # Should fall back to status_code # Test case 6: Exception with 'code' as None - should fall back to 'status_code' - none_code_exception = BothAttributesException( - code=None, status_code=401, message="Unauthorized" - ) + none_code_exception = BothAttributesException(code=None, status_code=401, message="Unauthorized") result = StandardLoggingPayloadSetup.get_error_information(none_code_exception) assert result["error_code"] == "401" # Should fall back to status_code @@ -2859,9 +2740,7 @@ def test_get_error_information_prefers_message_attribute_over_str(): ) result = StandardLoggingPayloadSetup.get_error_information(exc) - assert ( - result["error_message"] == msg - ), f"expected message from .message attribute, got {result['error_message']!r}" + assert result["error_message"] == msg, f"expected message from .message attribute, got {result['error_message']!r}" assert result["error_code"] == "401" assert result["error_class"] == "ProxyExceptionLike" @@ -2936,8 +2815,7 @@ def test_get_error_information_preserves_explicit_empty_message(): exc = ProxyExceptionLike(message="", code=500) result = StandardLoggingPayloadSetup.get_error_information(exc) assert result["error_message"] == "", ( - "explicit empty .message must survive verbatim; got " - f"{result['error_message']!r}" + f"explicit empty .message must survive verbatim; got {result['error_message']!r}" ) @@ -3200,9 +3078,7 @@ def test_process_hidden_params_recalculates_cost_after_failure_handler_zero(): choices=[{"message": {"role": "assistant", "content": "ok"}}], usage=Usage(prompt_tokens=9698, completion_tokens=30, total_tokens=9728), ) - logging_obj._process_hidden_params_and_response_cost( - result, datetime.now(), datetime.now() - ) + logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now()) cost = logging_obj.model_call_details.get("response_cost") assert cost is not None and cost > 0 @@ -3226,9 +3102,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params(): litellm_call_id="test-hidden-zero-cost", function_id="test-hidden-zero-cost", ) - logging_obj.model_call_details["litellm_params"] = { - "model": "gemini-2.5-flash-lite" - } + logging_obj.model_call_details["litellm_params"] = {"model": "gemini-2.5-flash-lite"} logging_obj.optional_params = {} result = ModelResponse( @@ -3238,9 +3112,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params(): ) result._hidden_params = {"response_cost": 0.0} - logging_obj._process_hidden_params_and_response_cost( - result, datetime.now(), datetime.now() - ) + logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now()) assert logging_obj.model_call_details.get("response_cost") == 0.0 slo = logging_obj.model_call_details.get("standard_logging_object") or {} @@ -3289,9 +3161,7 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer ) result._hidden_params = {"response_cost": passthrough_cost} - logging_obj._process_hidden_params_and_response_cost( - result, datetime.now(), datetime.now() - ) + logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now()) assert logging_obj.model_call_details.get("response_cost") == passthrough_cost slo = logging_obj.model_call_details.get("standard_logging_object") or {} @@ -3348,9 +3218,7 @@ def test_function_setup_litellm_metadata_populates_metadata(): assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash # metadata should be a COPY, not an alias — mutating one must not affect the other - assert ( - metadata is not litellm_metadata - ), "litellm_params['metadata'] should be a copy, not the same object" + assert metadata is not litellm_metadata, "litellm_params['metadata'] should be a copy, not the same object" def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup(): @@ -3395,9 +3263,9 @@ def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup(): litellm_params = logging_obj.model_call_details.get("litellm_params", {}) litellm_metadata = litellm_params.get("litellm_metadata") assert litellm_metadata is not None - assert litellm_metadata.get("standard_logging_guardrail_information") == [ - guardrail_entry - ], "guardrail writes after function_setup must be visible to the logging object" + assert litellm_metadata.get("standard_logging_guardrail_information") == [guardrail_entry], ( + "guardrail writes after function_setup must be visible to the logging object" + ) assert litellm_metadata.get("applied_guardrails") == ["pam-ethical-request"] merged = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) @@ -3566,9 +3434,7 @@ def test_failure_handler_skips_sync_callbacks_for_pass_through_requests(logging_ @pytest.mark.parametrize("call_type", ["completion", "acompletion"]) -def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests( - logging_obj, call_type -): +def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(logging_obj, call_type): """Ensure sync failure callbacks still fire for normal (non-pass-through) requests.""" from litellm.integrations.custom_logger import CustomLogger @@ -3729,9 +3595,7 @@ def test_standard_logging_hidden_params_backfills_response_cost_without_mutating ) response._hidden_params = {"response_cost": None, "model_id": "mid-test"} - payload = logging_obj._build_standard_logging_payload( - response, datetime.now(), datetime.now() - ) + payload = logging_obj._build_standard_logging_payload(response, datetime.now(), datetime.now()) assert payload is not None assert payload["hidden_params"]["response_cost"] == 0.002 @@ -3785,10 +3649,7 @@ def test_merge_hidden_params_from_response_into_metadata_no_op_when_empty(): _hidden_params = {} logging_obj._merge_hidden_params_from_response_into_metadata(_NoHp()) - assert ( - "hidden_params" - not in logging_obj.model_call_details["litellm_params"]["metadata"] - ) + assert "hidden_params" not in logging_obj.model_call_details["litellm_params"]["metadata"] # ── StandardLoggingPayloadSetup.get_additional_headers ─────────────────────── @@ -3985,9 +3846,7 @@ def test_success_handler_computes_cost_for_dict_response(): "_build_standard_logging_payload", return_value={"response_cost": expected_cost}, ), - patch( - "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload" - ), + patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"), patch.object( logging_obj, "_is_recognized_call_type_for_logging", @@ -4024,9 +3883,7 @@ def test_success_handler_preserves_precomputed_cost_for_dict_response(): "_build_standard_logging_payload", return_value={"response_cost": precomputed_cost}, ), - patch( - "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload" - ), + patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"), patch.object( logging_obj, "_is_recognized_call_type_for_logging", @@ -4065,9 +3922,7 @@ def test_success_handler_unified_helper_runs_for_typed_results(): "_build_standard_logging_payload", return_value={"response_cost": expected_cost}, ), - patch( - "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload" - ), + patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"), patch.object( logging_obj, "_is_recognized_call_type_for_logging", @@ -4122,9 +3977,7 @@ class TestFirstApiCallStartTimeSetOnce: assert first == obj.model_call_details["api_call_start_time"] # Set on the logging object only — user metadata untouched. assert user_meta == {} - assert ( - "first_api_call_start_time" not in obj.model_call_details["litellm_params"] - ) + assert "first_api_call_start_time" not in obj.model_call_details["litellm_params"] time.sleep(0.002) # ensure a distinct retry timestamp obj.pre_call(input="hi", api_key="sk-test") @@ -4141,18 +3994,16 @@ def test_get_error_information_for_logging_payload_ignores_spoofed_disconnect_wi baseline = StandardLoggingPayloadSetup.get_error_information( original_exception=ValueError("provider failure"), ) - error_information, error_str = ( - StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata={ - "error_information": { - "error_code": "499", - "error_message": "Client disconnected the request", - "error_class": "ClientDisconnected", - } - }, - original_exception=ValueError("provider failure"), - error_str="provider failure", - ) + error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata={ + "error_information": { + "error_code": "499", + "error_message": "Client disconnected the request", + "error_class": "ClientDisconnected", + } + }, + original_exception=ValueError("provider failure"), + error_str="provider failure", ) assert error_information == baseline assert error_str == "provider failure" @@ -4166,22 +4017,18 @@ def test_get_error_information_for_logging_payload_client_disconnect(): "error_message": "Client disconnected the request", "error_class": "ClientDisconnected", } - error_information, error_str = ( - StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata={"client_disconnected": True, "error_information": custom_error}, - original_exception=None, - error_str=None, - ) + error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata={"client_disconnected": True, "error_information": custom_error}, + original_exception=None, + error_str=None, ) assert error_information == custom_error assert error_str == "Client disconnected the request" - error_information, error_str = ( - StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata={"client_disconnected": True}, - original_exception=None, - error_str="existing error", - ) + error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata={"client_disconnected": True}, + original_exception=None, + error_str="existing error", ) assert error_information["error_code"] == "499" assert error_str == "existing error" @@ -4189,12 +4036,10 @@ def test_get_error_information_for_logging_payload_client_disconnect(): baseline = StandardLoggingPayloadSetup.get_error_information( original_exception=None, ) - error_information, error_str = ( - StandardLoggingPayloadSetup.get_error_information_for_logging_payload( - metadata={}, - original_exception=None, - error_str=None, - ) + error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( + metadata={}, + original_exception=None, + error_str=None, ) assert error_information == baseline assert error_str is None @@ -4229,9 +4074,7 @@ def test_get_error_information_prefers_message_attribute_over_empty_str(): def __str__(self): return "" - info = StandardLoggingPayloadSetup.get_error_information( - original_exception=_SilentExc() - ) + info = StandardLoggingPayloadSetup.get_error_information(original_exception=_SilentExc()) assert info["error_message"] == "real failure detail" assert info["error_code"] == "401" @@ -4262,9 +4105,7 @@ def _responses_api_response_with_text(text="hello world"): type="message", role="assistant", status="completed", - content=[ - ResponseOutputText(annotations=[], text=text, type="output_text") - ], + content=[ResponseOutputText(annotations=[], text=text, type="output_text")], ) ], usage=ResponseAPIUsage(input_tokens=11, output_tokens=7, total_tokens=18), @@ -4279,9 +4120,7 @@ def _responses_api_response_with_text(text="hello world"): ("ResponseFailedEvent", "response.failed"), ], ) -def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event( - event_cls, event_type -): +def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(event_cls, event_type): """Regression for #28595 / #28943. When anthropic_messages routes to the OpenAI Responses backend and stream=True, success_handler receives a terminal Responses API event. The handler must translate it to a ModelResponse whose choices carry @@ -4320,10 +4159,7 @@ def test_handle_anthropic_messages_response_logging_passes_model_response_throug """Anthropic-native path already yields a ModelResponse; it must be returned unchanged.""" logging_obj = _anthropic_messages_logging_obj() model_response = ModelResponse() - assert ( - logging_obj._handle_anthropic_messages_response_logging(result=model_response) - is model_response - ) + assert logging_obj._handle_anthropic_messages_response_logging(result=model_response) is model_response def test_handle_anthropic_messages_response_logging_degrades_on_unparseable_responses_payload(): @@ -4619,9 +4455,7 @@ def test_non_image_response_has_no_output_image_count(logging_obj): def test_zero_token_video_usage_preserves_duration_seconds(logging_obj): """Video usage bills by duration; the payload must keep duration_seconds even with zero tokens.""" - payload = _build_payload_for_media_response( - logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}} - ) + payload = _build_payload_for_media_response(logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}}) assert payload is not None assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0 @@ -5782,3 +5616,93 @@ def test_failure_handler_helper_fn_builds_payload_once_per_exception(): other_exc = _raise_and_catch(_ClientError(status_code=429, message="rate limited")) obj._failure_handler_helper_fn(exception=other_exc, traceback_exception="") assert obj.model_call_details["standard_logging_object"] is not first_payload + + +@pytest.mark.asyncio +async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_obj): + """The savings gate reads litellm_gateway_injected_cache from the request's + metadata bucket. Recording lives in the shared prompt-hook wrappers, so chat, + /v1/responses, router prompt deployments, and proxy prompt templates all mark + injected requests the same way; a hook that injects nothing leaves no marker.""" + from litellm.integrations.custom_prompt_management import CustomPromptManagement + + class _InjectingHook(CustomPromptManagement): + def get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id, + prompt_variables, + dynamic_callback_params, + prompt_label=None, + prompt_version=None, + prompt_spec=None, + ): + marked = [{**messages[0], "cache_control": {"type": "ephemeral"}}, *messages[1:]] + return model, marked, non_default_params + + async def async_get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id, + prompt_variables, + dynamic_callback_params, + litellm_logging_obj=None, + tools=None, + prompt_label=None, + prompt_version=None, + prompt_spec=None, + ): + return self.get_chat_completion_prompt( + model, messages, non_default_params, prompt_id, prompt_variables, dynamic_callback_params + ) + + class _PassthroughHook(CustomPromptManagement): + def get_chat_completion_prompt( + self, + model, + messages, + non_default_params, + prompt_id, + prompt_variables, + dynamic_callback_params, + prompt_label=None, + prompt_version=None, + prompt_spec=None, + ): + return model, messages, non_default_params + + request_kwargs = {"metadata": {}, "model_info": {"id": "dep-of-this-attempt"}} + _, marked, _ = await logging_obj.async_get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_InjectingHook(), + request_kwargs=request_kwargs, + ) + assert request_kwargs["metadata"]["litellm_gateway_injected_cache"] == "dep-of-this-attempt" + + logging_obj.get_chat_completion_prompt( + model="claude-sonnet-5", + messages=marked, + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_PassthroughHook(), + request_kwargs=request_kwargs, + ) + assert request_kwargs["metadata"]["litellm_gateway_injected_cache"] == "dep-of-this-attempt" + + untouched = {"metadata": {}} + logging_obj.get_chat_completion_prompt( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + non_default_params={}, + prompt_variables=None, + prompt_management_logger=_PassthroughHook(), + request_kwargs=untouched, + ) + assert "litellm_gateway_injected_cache" not in untouched["metadata"] diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index a00815345aa..681132105ad 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -207,6 +207,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, } @@ -258,6 +259,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, } diff --git a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py index c2d0f64461a..c1efb3e7220 100644 --- a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py @@ -85,10 +85,10 @@ def test_one_statement_carries_every_row_in_the_batch(): assert sql.count("INSERT INTO") == 1 assert len(re.findall(r"ON CONFLICT", sql)) == 1 - # 22 bound columns per row plus the inlined updated_at, so the row count is what + # 23 bound columns per row plus the inlined updated_at, so the row count is what # separates one multi-row statement from a hundred single-row ones. - assert len(params) == 100 * 22 - assert "$2200::text" in sql + assert len(params) == 100 * 23 + assert "$2300::text" in sql assert sql.count("(NOW() AT TIME ZONE 'UTC')") == 100 + 1 diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index ca1827aa38e..b1f647bdd3d 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2331,6 +2331,7 @@ async def test_daily_transaction_carries_compression_saved_tokens(): metadata = { "usage_object": {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 15}, + "litellm_gateway_injected_cache": "dep-of-the-compression-row", "compression_savings": { "tokens_before": 12000, "tokens_after": 5000, @@ -2354,6 +2355,7 @@ async def test_daily_transaction_carries_compression_saved_tokens(): "model": "claude-sonnet-5", "custom_llm_provider": "anthropic", "model_group": "claude-sonnet-5", + "model_id": "dep-of-the-compression-row", "call_type": "anthropic_messages", "prompt_tokens": 5000, "completion_tokens": 10, @@ -2740,3 +2742,105 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_on_responses_calls( assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}] assert PrismaClient.spend_log_flush_requested.is_set() is expects_flush PrismaClient.spend_log_flush_requested.clear() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "injected_deployment, attributed", + [ + pytest.param("dep-of-this-row", True, id="this-deployment-injected"), + pytest.param("dep-of-a-sibling-leg", False, id="a-sibling-deployment-injected"), + pytest.param("", True, id="injected-before-a-deployment-was-chosen"), + ], +) +async def test_caching_savings_are_attributed_to_the_deployment_that_was_injected( + injected_deployment, attributed +): + """Retries, same-group failover and cross-model-group fallbacks all reuse one metadata + bucket and one litellm_call_id, so a marker written by the leg that injected is + visible to every sibling and nothing request-scoped can tell them apart. + + Naming the deployment it injected for is what keeps the credit on that leg: a row + billed for a different deployment reads it as no injection, so no seam has to strip + it and a deployment that injected nothing is never credited for the one that did. + + An injection that ran before any deployment was chosen, which is what the proxy does + for prompt templates, is written into the payload every leg goes on to send, so it + marks the request for all of them and each leg keeps the credit. + """ + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + payload = { + "request_id": "req-fallback-leg", + "user": "test-user", + "startTime": "2026-07-17T00:00:00", + "api_key": "test-key", + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "model_group": "claude-sonnet-5", + "model_id": "dep-of-this-row", + "call_type": "anthropic_messages", + "prompt_tokens": 5000, + "completion_tokens": 10, + "spend": 0.05, + "metadata": json.dumps( + { + "usage_object": {"cache_read_input_tokens": 4242, "cache_creation_input_tokens": 1111}, + "litellm_gateway_injected_cache": injected_deployment, + } + ), + } + + transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=payload, + prisma_client=mock_prisma, + type="user", + ) + + assert transaction is not None + assert transaction["prompt_caching_savings_spend"] != 0.0 + assert (transaction["gateway_injected_caching_savings_spend"] != 0.0) is attributed + + +@pytest.mark.asyncio +async def test_daily_transaction_attributes_caching_savings_only_with_an_injection_marker(): + """Cached usage with no litellm_gateway_injected_cache marker is still a real saving. + + Client-sent cache_control and implicit provider caching leave no marker, so the row + keeps the total the customer actually got while the gateway-attributed column stays + empty, which is what separates what caching saved from what litellm can claim. + """ + writer = DBSpendUpdateWriter() + mock_prisma = MagicMock() + mock_prisma.get_request_status = MagicMock(return_value="success") + + payload = { + "request_id": "req-ungated-caching", + "user": "test-user", + "startTime": "2026-07-17T00:00:00", + "api_key": "test-key", + "model": "claude-sonnet-5", + "custom_llm_provider": "anthropic", + "model_group": "claude-sonnet-5", + "call_type": "anthropic_messages", + "prompt_tokens": 5000, + "completion_tokens": 10, + "spend": 0.05, + "metadata": json.dumps( + {"usage_object": {"cache_read_input_tokens": 4242, "cache_creation_input_tokens": 1111}} + ), + } + + transaction = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload=payload, + prisma_client=mock_prisma, + type="user", + ) + + assert transaction is not None + assert transaction["cache_read_input_tokens"] == 4242 + assert transaction["cache_creation_input_tokens"] == 1111 + assert transaction["prompt_caching_savings_spend"] != 0.0 + assert transaction["gateway_injected_caching_savings_spend"] == 0.0 diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 1bcb331430e..a258127acff 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -155,6 +155,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, "failed_requests": 0, } @@ -485,6 +486,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.compression_saved_tokens = 0 mock_record_1.compression_savings_spend = 0.0 mock_record_1.prompt_caching_savings_spend = 0.0 + mock_record_1.gateway_injected_caching_savings_spend = 0.0 mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 @@ -508,6 +510,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.compression_saved_tokens = 0 mock_record_2.compression_savings_spend = 0.0 mock_record_2.prompt_caching_savings_spend = 0.0 + mock_record_2.gateway_injected_caching_savings_spend = 0.0 mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 @@ -571,6 +574,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, "failed_requests": 0, } @@ -657,6 +661,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, api_requests=1, successful_requests=1, @@ -1089,6 +1094,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "compression_saved_tokens": None, "compression_savings_spend": None, "prompt_caching_savings_spend": None, + "gateway_injected_caching_savings_spend": None, "autorouter_savings_spend": None, "api_requests": None, "successful_requests": None, @@ -1133,6 +1139,7 @@ def _no_spend_record(): compression_saved_tokens=None, compression_savings_spend=None, prompt_caching_savings_spend=None, + gateway_injected_caching_savings_spend=None, autorouter_savings_spend=None, api_requests=None, successful_requests=None, @@ -1242,6 +1249,7 @@ def _spend_record(api_key, *, model="gpt-4o-mini-ptu", spend=0.0, ptu_flat_cost= compression_saved_tokens=0, compression_savings_spend=0, prompt_caching_savings_spend=0, + gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, total_tokens=0, api_requests=0, @@ -1307,6 +1315,7 @@ def _grouping_row( compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + gateway_injected_caching_savings_spend=0.0, autorouter_savings_spend=0.0, api_requests=0, successful_requests=0, @@ -1466,6 +1475,7 @@ def test_update_breakdown_metrics_covers_mcp_endpoint_and_entity(ptu_cost_attrib compression_saved_tokens=0, compression_savings_spend=0, prompt_caching_savings_spend=0, + gateway_injected_caching_savings_spend=0, autorouter_savings_spend=0, total_tokens=0, api_requests=0, @@ -1869,6 +1879,7 @@ async def test_get_daily_activity_aggregated_with_entity_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "gateway_injected_caching_savings_spend": 0.0, "autorouter_savings_spend": 0.0, "failed_requests": 0, "prompt_tokens": 0, diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index bb8345a9142..bc9e2a0f7cc 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -8,6 +8,7 @@ from litellm.proxy.spend_tracking.savings import ( _baseline_usage, compute_autorouter_savings, compute_savings_spend, + marks_gateway_injection, ) from litellm.router import Router from litellm.types.utils import Usage @@ -59,6 +60,7 @@ def test_compression_savings_priced_at_input_rate(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=4389, + gateway_injected_cache=True, ) assert result.compression == pytest.approx(4389 * input_cost) assert result.compression > 0 @@ -74,6 +76,7 @@ def test_prompt_caching_savings_priced_at_input_minus_cache_read(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 8200}, ) assert result.prompt_caching == pytest.approx(8200 * (input_cost - cache_read_cost)) @@ -126,6 +129,7 @@ def test_prompt_caching_savings_nets_out_the_cache_write_premium(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=usage_object, ) assert result.prompt_caching == pytest.approx(_net_caching_savings_against_biller(usage_object)) @@ -141,6 +145,7 @@ def test_prompt_caching_savings_go_negative_on_a_write_only_request(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=usage_object, ) true_savings = _net_caching_savings_against_biller(usage_object) @@ -156,6 +161,7 @@ def test_prompt_caching_savings_negative_when_writes_outweigh_reads(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=usage_object, ) true_savings = _net_caching_savings_against_biller(usage_object) @@ -172,6 +178,7 @@ def test_read_only_request_is_unchanged_by_the_write_premium(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=20000, written=0), ) assert result.prompt_caching == pytest.approx(20000 * (input_cost - cache_read_cost)) @@ -185,12 +192,14 @@ def test_openai_style_cache_write_tokens_are_netted_out(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 5000, "cache_creation_input_tokens": 800}, ) nested_only = compute_savings_spend( model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object={ "prompt_tokens_details": {"cached_tokens": 5000, "cache_write_tokens": 800}, }, @@ -221,6 +230,7 @@ def test_model_without_a_cache_write_price_takes_no_premium(): model=model, custom_llm_provider=None, compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=5000, written=5000), ) assert result.prompt_caching == pytest.approx(5000 * (input_cost - cache_read_cost)) @@ -244,6 +254,7 @@ def test_zero_cache_write_price_is_read_as_unpublished(): model="deepseek-chat", custom_llm_provider="deepseek", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=0, written=10000), ) assert result.prompt_caching == pytest.approx(0.0) @@ -268,6 +279,7 @@ def test_zero_cache_read_price_stays_literal(): model=model, custom_llm_provider=None, compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=10000, written=0), ) # free reads => the whole input rate is saved, not zero @@ -293,6 +305,7 @@ def test_sub_input_cache_write_price_is_an_extra_saving(): model=model, custom_llm_provider=None, compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=1000, written=4000), ) assert result.prompt_caching == pytest.approx(4000 * (input_cost - cheap_write)) @@ -306,6 +319,7 @@ def test_negative_cache_write_count_clamps_to_zero(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 1000, "cache_creation_input_tokens": -5000}, ) assert result.prompt_caching == pytest.approx(1000 * (input_cost - cache_read_cost)) @@ -316,6 +330,7 @@ def test_unknown_model_fails_open_to_zero(): model="totally-made-up-model-xyz", custom_llm_provider="anthropic", compression_saved_tokens=1000, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 1000}, ) assert result.compression == 0.0 @@ -327,6 +342,7 @@ def test_missing_model_fails_open_to_zero(): model=None, custom_llm_provider=None, compression_saved_tokens=1000, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 1000}, ) assert result.compression == 0.0 @@ -338,6 +354,7 @@ def test_negative_token_counts_clamp_to_zero(): model="claude-sonnet-5", custom_llm_provider="anthropic", compression_saved_tokens=-500, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": -500}, ) assert result.compression == 0.0 @@ -516,6 +533,7 @@ def test_autorouter_savings_zero_without_baseline(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision=None, usage_object=_cached_usage_object(), ) @@ -530,6 +548,7 @@ def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision={"conversation_continuing": True}, usage_object=_cached_usage_object(), ) @@ -543,6 +562,7 @@ def test_the_driver_is_off_until_a_baseline_is_configured(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=1000, + gateway_injected_cache=True, routing_decision={"conversation_continuing": True}, usage_object=_cached_usage_object(), ) @@ -557,6 +577,7 @@ def test_malformed_usage_object_does_not_fail_the_spend_write(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=1000, + gateway_injected_cache=True, routing_decision={"conversation_continuing": True}, usage_object={"prompt_tokens": ["not", "a", "number"]}, ) @@ -573,6 +594,7 @@ def test_model_without_cache_read_pricing_yields_no_caching_savings(): model=model, custom_llm_provider="azure", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object={"cache_read_input_tokens": 5000}, ) assert result.prompt_caching == 0.0 @@ -882,6 +904,7 @@ def test_a_baseline_recorded_on_the_decision_turns_the_driver_on(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"}, usage_object=_cached_usage_object(), ) @@ -895,6 +918,7 @@ def test_the_configured_baseline_overrides_the_recorded_one(monkeypatch): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision={ "conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5", @@ -923,6 +947,7 @@ def test_a_non_string_recorded_baseline_is_ignored(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision={"conversation_continuing": True, "savings_baseline_model": ["anthropic/claude-opus-5"]}, usage_object=_cached_usage_object(), ) @@ -954,6 +979,7 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one(): model="claude-sonnet-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=1000, written=20000), model_id=deployment_id, llm_router=lambda: router, @@ -965,6 +991,7 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one(): model="claude-sonnet-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, usage_object=_caching_usage(read=1000, written=20000), ) assert result.prompt_caching > at_public_rates.prompt_caching @@ -995,6 +1022,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision=decision, usage_object=_cached_usage_object(), llm_router=lambda: router, @@ -1003,6 +1031,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=True, routing_decision={k: v for k, v in decision.items() if k != "savings_baseline_deployment_id"}, usage_object=_cached_usage_object(), llm_router=lambda: router, @@ -1021,6 +1050,7 @@ def test_recorded_savings_win_over_recomputation(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=False, routing_decision=_routed_decision(), usage_object=_cached_usage_object(), recorded_autorouter_savings=0.5, @@ -1035,6 +1065,7 @@ def test_recorded_savings_survive_an_unusable_usage_object(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=False, routing_decision=_routed_decision(), usage_object={"prompt_tokens": ["not", "a", "number"]}, recorded_autorouter_savings=0.25, @@ -1047,6 +1078,7 @@ def test_a_boolean_is_not_a_recorded_savings_figure(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=False, routing_decision=None, usage_object=_cached_usage_object(), recorded_autorouter_savings=True, @@ -1063,6 +1095,7 @@ def test_rows_written_before_the_field_shipped_recompute(): model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, + gateway_injected_cache=False, routing_decision=_routed_decision(), usage_object=_cached_usage_object(), ) @@ -1127,3 +1160,95 @@ def test_logging_payload_never_stamps_internal_calls(): cost_breakdown=None, ) assert internal is None + + +def test_caching_savings_require_a_gateway_injected_breakpoint(): + """The same cached usage is attributed to the gateway only when it added a breakpoint. + + Client-sent cache_control and implicit provider caching (OpenAI, Gemini) produce + cache reads the gateway had no hand in. Those still count as caching savings the + customer really got, so the total is unchanged, but nothing about them is the + gateway's doing and the attributed figure has to stay empty. + """ + input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") + credited = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=True, + usage_object=_caching_usage(read=8200, written=0), + ) + expected = 8200 * (input_cost - cache_read_cost) + assert credited.prompt_caching == pytest.approx(expected) + assert credited.gateway_injected_caching == pytest.approx(expected) + unattributed = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=False, + usage_object=_caching_usage(read=8200, written=0), + ) + assert unattributed.prompt_caching == pytest.approx(expected) + assert unattributed.gateway_injected_caching == 0.0 + + +def test_unattributed_write_only_request_still_reports_its_loss_in_the_total(): + """A write-only request really did cost more than not caching, whoever asked for it. + + The attributed figure drops it because the gateway added no breakpoint, and dropping a + negative is why the attributed number can sit above the total rather than below it. + """ + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=False, + usage_object=_caching_usage(read=0, written=20000), + ) + assert result.prompt_caching < 0 + assert result.gateway_injected_caching == 0.0 + assert result.gateway_injected_caching > result.prompt_caching + + +def test_injected_request_keeps_its_negative_net(): + """A gateway-injected write-heavy request still reports its real loss.""" + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=True, + usage_object=_caching_usage(read=0, written=20000), + ) + assert result.prompt_caching < 0 + + +def test_attribution_does_not_touch_compression_or_autorouter_legs(): + input_cost, _ = _anthropic_costs("claude-sonnet-5") + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=4389, + gateway_injected_cache=False, + usage_object=_caching_usage(read=8200, written=0), + ) + assert result.compression == pytest.approx(4389 * input_cost) + assert result.prompt_caching > 0 + assert result.gateway_injected_caching == 0.0 + + +def test_marks_gateway_injection_credits_only_the_deployment_that_was_injected(): + """Every retry, failover and fallback of a request shares one metadata bucket and one + litellm_call_id, so the deployment is what tells those legs apart. A marker naming a + sibling has to read here as no injection; that is what keeps the credit on the leg + that earned it without any seam having to strip it. Anything that is not this row's + own deployment, the missing key included, is fail-closed.""" + assert marks_gateway_injection(None, "dep-a") is False + assert marks_gateway_injection({}, "dep-a") is False + assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, "dep-a") is True + assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, "dep-b") is False + assert marks_gateway_injection({"litellm_gateway_injected_cache": "dep-a"}, None) is False + # injected before a deployment was chosen, so it is in the payload every leg sends + assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, "dep-a") is True + assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, None) is True + assert marks_gateway_injection({"litellm_call_id": "c1"}, "dep-a") is False + assert marks_gateway_injection({"litellm_gateway_injected_cache": True}, "dep-a") is False diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 8c15ead8983..4a37ae23ff4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2629,7 +2629,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2725,7 +2725,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -2819,7 +2819,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 5b9d591ea56..87841720e43 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -3839,3 +3839,44 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully(): assert ( metadata.get("original_model_group") is None ), "original_model_group should be None when not provided" +@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) +def test_injected_cache_breakpoints_survive_into_spend_log_metadata(bucket): + """The injection marker only gates savings if it reaches the spend-log row. + + _get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an + undeclared key is dropped silently. Both buckets are covered because chat routes + stamp metadata while /v1/messages routes stamp litellm_metadata, and + record_gateway_injection writes into whichever the request carries. + """ + payload = get_logging_payload( + kwargs={ + "model": "claude-sonnet-5", + "litellm_params": { + bucket: { + "user_api_key": "test-key", + "litellm_gateway_injected_cache": "dep-of-this-row", + } + }, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-injected", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["litellm_gateway_injected_cache"] == "dep-of-this-row" + + +def test_passthrough_caching_carries_no_injection_marker(): + """The negative class the gate depends on: a request whose cache_control the client + supplied must read as unmarked, not merely unlabelled by accident.""" + payload = get_logging_payload( + kwargs={ + "model": "claude-sonnet-5", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-passthrough", choices=[], usage=litellm.Usage()), + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + metadata = json.loads(payload["metadata"]) + assert metadata["litellm_gateway_injected_cache"] is None diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 50ef6f29ec2..d13a2de94d0 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -920,6 +920,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "applied_policies": ["spoofed-policy"], "policy_sources": {"spoofed-policy": "request"}, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, + "litellm_gateway_injected_cache": "forged-deployment-id", "_session_deployment_affinity_ttl": 999999, "internal_call_origin": "autorouter_classifier", "_guardrail_pipelines": [{"name": "spoofed"}], @@ -934,6 +935,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "disable_global_guardrails": True, "enable_prompt_caching": True, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, + "litellm_gateway_injected_cache": "forged-deployment-id", "metadata": copy.deepcopy(malicious_metadata), "litellm_metadata": copy.deepcopy(malicious_metadata), } @@ -952,6 +954,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "disable_global_guardrails" not in updated assert "enable_prompt_caching" not in updated assert "routing_decision" not in updated + assert "litellm_gateway_injected_cache" not in updated stripped_keys = { "disable_global_guardrails", @@ -966,6 +969,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "applied_policies", "policy_sources", "routing_decision", + "litellm_gateway_injected_cache", "_session_deployment_affinity_ttl", "internal_call_origin", "_guardrail_pipelines", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 7f075e48341..ba40163a416 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -17,6 +17,7 @@ export const shortDate = (iso: string): string => export const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; export const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +export const gatewayAttributedCachingOf = (m: SpendMetrics): number => m.gateway_injected_caching_savings_spend ?? 0; export const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0; export const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index 420977f8c31..8e7c1869df2 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -12,6 +12,7 @@ export interface SpendMetrics { compression_saved_tokens?: number; compression_savings_spend?: number; prompt_caching_savings_spend?: number; + gateway_injected_caching_savings_spend?: number; autorouter_savings_spend?: number; } diff --git a/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx b/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx index 64294e05ef3..f3c9c64997b 100644 --- a/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx +++ b/ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx @@ -6,6 +6,7 @@ import SummaryCard from "@/components/shared/SummaryCard"; import { autorouterOf, cachingOf, + gatewayAttributedCachingOf, compressionOf, savedTokensOf, usd, @@ -25,6 +26,7 @@ export const useSavingsTotals = (results: DailyData[]) => compression, caching, autorouter, + gatewayAttributedCaching: sumOf(gatewayAttributedCachingOf), savedTokens: sumOf(savedTokensOf), total: compression + caching + autorouter, }; @@ -34,7 +36,7 @@ const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading: const totals = useSavingsTotals(results); return ( -
+
+