scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(spend): track prompt compression saved tokens in daily spend aggregates

Browse source 
Native compression interception now records tokens_before/after/saved into the
request litellm_metadata so savings land in the SpendLog metadata JSON under a
typed compression_savings key. A single normalizer
(extract_compression_saved_tokens) sums that key with Headroom guardrail
tokens_saved; the two writers are disjoint and run at different stages, so
summing never double-counts. The spend-log redactor now preserves purely
numeric compression stats inside guardrail_response so Headroom savings
survive the store_prompts_in_spend_logs=false default. compression_saved_tokens
is threaded through BaseDailySpendTransaction, queue aggregation, the daily
upsert blocks, a new BigInt column on all six daily spend tables, and the
daily activity read path (SpendMetrics, DailySpendMetadata, raw-SQL rollups)
This commit is contained in:
Tin Chi Lo 2026-07-17 21:32:49 -07:00
parent 3829fa3014
commit c91c6d0e85
22 changed files with 731 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
litellm-proxy-extras/litellm_proxy_extras/migrations/20260717000000_add_compression_saved_tokens/migration.sql Normal file
View file

@ -0,0 +1,17 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "compression_saved_tokens" BIGINT NOT NULL DEFAULT 0;

6
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -729,6 +729,7 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -760,6 +761,7 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -791,6 +793,7 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -821,6 +824,7 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -851,6 +855,7 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -883,6 +888,7 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

42
litellm/integrations/compression_interception/handler.py
View file

@ -14,6 +14,7 @@ from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
CompressionSavingsMetadata,
)
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
@ -25,6 +26,41 @@ LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve"
_CACHE_TTL_SECONDS = 15 * 60
def _compression_savings_from_counts(
original_tokens: object, compressed_tokens: object
) -> CompressionSavingsMetadata | None:
if isinstance(original_tokens, bool) or not isinstance(original_tokens, int):
return None
if isinstance(compressed_tokens, bool) or not isinstance(compressed_tokens, int):
return None
if compressed_tokens < 0 or original_tokens < compressed_tokens:
return None
return CompressionSavingsMetadata(
tokens_before=original_tokens,
tokens_after=compressed_tokens,
tokens_saved=original_tokens - compressed_tokens,
source="compression_interception",
)
def _record_compression_savings(kwargs: dict[str, object], savings: CompressionSavingsMetadata) -> None:
"""
Attach savings to the request's litellm metadata so they land in the
SpendLog row's metadata JSON under ``compression_savings``.
``/v1/messages`` requests carry proxy metadata under ``litellm_metadata``
(the ``metadata`` key is Anthropic's own API field). The existing dict is
updated in place because the proxy and the logging object hold references
to the same object; replacing it would orphan writes made through those
references.
"""
existing = kwargs.get("litellm_metadata")
if isinstance(existing, dict):
existing["compression_savings"] = savings
return
kwargs["litellm_metadata"] = {"compression_savings": savings}
class CompressionInterceptionLogger(CustomLogger):
"""
CustomLogger that implements transparent prompt compression + retrieval loops.
@ -130,6 +166,12 @@ class CompressionInterceptionLogger(CustomLogger):
call_id = str(uuid.uuid4())
kwargs["litellm_call_id"] = call_id
self._compression_cache_by_call_id[call_id] = (cache, time.time())
savings = _compression_savings_from_counts(
original_tokens=compressed.get("original_tokens"),
compressed_tokens=compressed.get("compressed_tokens"),
)
if savings is not None:
_record_compression_savings(kwargs=kwargs, savings=savings)
verbose_logger.debug(
"CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]",
call_id,

5
litellm/proxy/_types.py
View file

@ -20,6 +20,9 @@ from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_no_callback_env_reference,
)
from litellm.types.integrations.compression_interception import (
CompressionSavingsMetadata,
)
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import (
AllMessageValues,
@ -3240,6 +3243,7 @@ class SpendLogsMetadata(TypedDict):
attempted_retries: Optional[int] # Number of retries attempted (0 = first attempt succeeded)
max_retries: Optional[int] # Max retries configured for this request
cost_breakdown: Optional[CostBreakdown] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
class SpendLogsPayload(TypedDict):
@ -4459,6 +4463,7 @@ class BaseDailySpendTransaction(TypedDict):
completion_tokens: int
cache_read_input_tokens: int
cache_creation_input_tokens: int
compression_saved_tokens: int
# request level metrics
spend: float

1
litellm/proxy/agent_endpoints/endpoints.py
View file

@ -1082,6 +1082,7 @@ async def get_agent_daily_activity(
total_failed_requests=0,
total_cache_read_input_tokens=0,
total_cache_creation_input_tokens=0,
total_compression_saved_tokens=0,
page=page,
total_pages=0,
has_more=False,

12
litellm/proxy/db/db_spend_update_writer.py
View file

@ -59,6 +59,9 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
ToolDiscoveryQueue,
)
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
if TYPE_CHECKING:
@ -1554,6 +1557,10 @@ class DBSpendUpdateWriter:
common_data["cache_creation_input_tokens"] = transaction.get(
"cache_creation_input_tokens", 0
)
if "compression_saved_tokens" in transaction:
common_data["compression_saved_tokens"] = transaction.get(
"compression_saved_tokens", 0
)
if entity_type == "tag" and "request_id" in transaction:
common_data["request_id"] = transaction.get("request_id")
@ -1577,6 +1584,10 @@ class DBSpendUpdateWriter:
update_data["cache_creation_input_tokens"] = {
"increment": transaction.get("cache_creation_input_tokens", 0)
}
if "compression_saved_tokens" in transaction:
update_data["compression_saved_tokens"] = {
"increment": transaction.get("compression_saved_tokens", 0)
}
if entity_type == "tag" and "request_id" in transaction:
update_data["request_id"] = transaction.get("request_id")
@ -1842,6 +1853,7 @@ class DBSpendUpdateWriter:
failed_requests=1 if request_status != "success" else 0,
cache_read_input_tokens=_extract_cache_read_tokens(usage_obj),
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
compression_saved_tokens=extract_compression_saved_tokens(_metadata),
)
return daily_transaction
except Exception as e:

4
litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py
View file

@ -122,6 +122,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
payload.get("cache_creation_input_tokens", 0) or 0
) + daily_transaction.get("cache_creation_input_tokens", 0)
daily_transaction["compression_saved_tokens"] = (
payload.get("compression_saved_tokens", 0) or 0
) + daily_transaction.get("compression_saved_tokens", 0)
else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
return aggregated_daily_spend_update_transactions

5
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -49,6 +49,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: Any) -> SpendMetrics:
existing_metrics.total_tokens += prompt_tokens + completion_tokens
existing_metrics.cache_read_input_tokens += record.cache_read_input_tokens or 0
existing_metrics.cache_creation_input_tokens += record.cache_creation_input_tokens or 0
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
existing_metrics.api_requests += record.api_requests or 0
existing_metrics.successful_requests += record.successful_requests or 0
existing_metrics.failed_requests += record.failed_requests or 0
@ -473,6 +474,7 @@ def _build_aggregated_sql_query(
SUM(completion_tokens)::bigint AS completion_tokens,
SUM(cache_read_input_tokens)::bigint AS cache_read_input_tokens,
SUM(cache_creation_input_tokens)::bigint AS cache_creation_input_tokens,
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
SUM(failed_requests)::bigint AS failed_requests
@ -612,6 +614,7 @@ def _record_to_spend_metrics(record: Any) -> SpendMetrics:
total_tokens=prompt_tokens + completion_tokens,
cache_read_input_tokens=record.cache_read_input_tokens or 0,
cache_creation_input_tokens=record.cache_creation_input_tokens or 0,
compression_saved_tokens=record.compression_saved_tokens or 0,
api_requests=record.api_requests or 0,
successful_requests=record.successful_requests or 0,
failed_requests=record.failed_requests or 0,
@ -862,6 +865,7 @@ async def get_daily_activity(
total_failed_requests=metadata_metrics.failed_requests,
total_cache_read_input_tokens=metadata_metrics.cache_read_input_tokens,
total_cache_creation_input_tokens=metadata_metrics.cache_creation_input_tokens,
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
has_more=(page * page_size) < total_count,
@ -948,6 +952,7 @@ async def get_daily_activity_aggregated(
total_failed_requests=aggregated["totals"].failed_requests,
total_cache_read_input_tokens=aggregated["totals"].cache_read_input_tokens,
total_cache_creation_input_tokens=aggregated["totals"].cache_creation_input_tokens,
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
page=1,
total_pages=1,
has_more=False,

6
litellm/proxy/schema.prisma
View file

@ -729,6 +729,7 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -760,6 +761,7 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -791,6 +793,7 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -821,6 +824,7 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -851,6 +855,7 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -883,6 +888,7 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

58
litellm/proxy/spend_tracking/compression_savings.py Normal file
View file

@ -0,0 +1,58 @@
"""
Single chokepoint for reading prompt-compression token savings out of a parsed
SpendLog ``metadata`` JSON dict. Imported by the daily-spend DB writer and by
cost-savings read endpoints.
"""
from collections.abc import Mapping
HEADROOM_GUARDRAIL_PROVIDER = "headroom"
def _saved_tokens_or_zero(value: object) -> int:
if isinstance(value, bool) or not isinstance(value, int):
return 0
if value < 0:
return 0
return value
def _tokens_saved_from_stats(stats: object) -> int:
if not isinstance(stats, Mapping):
return 0
return _saved_tokens_or_zero(stats.get("tokens_saved"))
def _headroom_entry_saved_tokens(entry: object) -> int:
if not isinstance(entry, Mapping):
return 0
if entry.get("guardrail_provider") != HEADROOM_GUARDRAIL_PROVIDER:
return 0
return _tokens_saved_from_stats(entry.get("guardrail_response"))
def _headroom_saved_tokens(guardrail_information: object) -> int:
if not isinstance(guardrail_information, list):
return 0
return sum(_headroom_entry_saved_tokens(entry) for entry in guardrail_information)
def extract_compression_saved_tokens(metadata: Mapping[str, object]) -> int:
"""
Return the total prompt tokens saved by compression for one request.
Sums two disjoint sources:
- the native ``compression_savings`` key, written only by
``CompressionInterceptionLogger`` in its pre-call deployment hook
- ``guardrail_information`` entries with ``guardrail_provider ==
"headroom"``, written only by the Headroom guardrail
Each writer records only its own transform pass and the two run at
different stages (guardrail pre-call vs deployment pre-call), so when both
fire on one request their measured savings are independent and additive;
summing them never double-counts. Malformed or missing values contribute 0.
"""
return _tokens_saved_from_stats(metadata.get("compression_savings")) + _headroom_saved_tokens(
metadata.get("guardrail_information")
)

34
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -115,6 +115,7 @@ def _get_spend_logs_metadata(
attempted_retries=None,
max_retries=None,
cost_breakdown=None,
compression_savings=None,
litellm_call_id=litellm_call_id,
)
verbose_proxy_logger.debug(
@ -903,14 +904,45 @@ _PROMPT_CARRYING_GUARDRAIL_FIELDS = (
"classification",
)
_NUMERIC_COMPRESSION_STAT_KEYS = (
"tokens_before",
"tokens_after",
"tokens_saved",
"compression_ratio",
)
def _numeric_compression_stats_from_guardrail_response(
guardrail_response: object,
) -> dict[str, int | float] | None:
if not isinstance(guardrail_response, dict):
return None
stats = {
key: value
for key, value in guardrail_response.items()
if key in _NUMERIC_COMPRESSION_STAT_KEYS and isinstance(value, (int, float)) and not isinstance(value, bool)
}
return stats or None
def _redact_prompt_fields_in_guardrail_entry(
entry: StandardLoggingGuardrailInformation,
) -> StandardLoggingGuardrailInformation:
return {
"""
Replace prompt-carrying fields with the redaction marker. Purely numeric
compression stats inside ``guardrail_response`` (e.g. Headroom's
``tokens_saved``) cannot carry prompt content, so they are preserved as a
stats-only dict; spend aggregation reads them via
``extract_compression_saved_tokens``.
"""
preserved_stats = _numeric_compression_stats_from_guardrail_response(entry.get("guardrail_response"))
redacted: StandardLoggingGuardrailInformation = {
**entry,
**{key: REDACTED_BY_LITELM_STRING for key in _PROMPT_CARRYING_GUARDRAIL_FIELDS if key in entry},
}
if preserved_stats is None:
return redacted
return {**redacted, "guardrail_response": preserved_stats}
def _sanitize_error_information_for_spend_logs(

14
litellm/types/integrations/compression_interception.py
View file

@ -2,7 +2,7 @@
Type definitions for Compression Interception integration.
"""
from typing import Any, Dict, Optional, TypedDict
from typing import Any, Dict, Literal, Optional, TypedDict
class CompressionInterceptionConfig(TypedDict, total=False):
@ -25,3 +25,15 @@ class CompressionInterceptionConfig(TypedDict, total=False):
compression_target: Optional[int]
embedding_model: Optional[str]
embedding_model_params: Optional[Dict[str, Any]]
class CompressionSavingsMetadata(TypedDict):
"""
Per-request prompt-compression savings recorded into the spend-log metadata
JSON so daily spend aggregates can track tokens saved by compression.
"""
tokens_before: int
tokens_after: int
tokens_saved: int
source: Literal["compression_interception"]

3
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -22,6 +22,7 @@ class SpendMetrics(BaseModel):
completion_tokens: int = Field(default=0)
cache_read_input_tokens: int = Field(default=0)
cache_creation_input_tokens: int = Field(default=0)
compression_saved_tokens: int = Field(default=0)
total_tokens: int = Field(default=0)
successful_requests: int = Field(default=0)
failed_requests: int = Field(default=0)
@ -79,6 +80,7 @@ class DailySpendMetadata(BaseModel):
total_failed_requests: int = Field(default=0)
total_cache_read_input_tokens: int = Field(default=0)
total_cache_creation_input_tokens: int = Field(default=0)
total_compression_saved_tokens: int = Field(default=0)
page: int = Field(default=1)
total_pages: int = Field(default=1)
has_more: bool = Field(default=False)
@ -102,6 +104,7 @@ class LiteLLM_DailyUserSpend(BaseModel):
completion_tokens: int = 0
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
compression_saved_tokens: int = 0
spend: float = 0.0
api_requests: int = 0
successful_requests: int = 0

6
schema.prisma
View file

@ -729,6 +729,7 @@ model LiteLLM_DailyUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -760,6 +761,7 @@ model LiteLLM_DailyOrganizationSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -791,6 +793,7 @@ model LiteLLM_DailyEndUserSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -821,6 +824,7 @@ model LiteLLM_DailyAgentSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -851,6 +855,7 @@ model LiteLLM_DailyTeamSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -883,6 +888,7 @@ model LiteLLM_DailyTagSpend {
completion_tokens BigInt @default(0)
cache_read_input_tokens BigInt @default(0)
cache_creation_input_tokens BigInt @default(0)
compression_saved_tokens BigInt @default(0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

127
tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py
View file

@ -396,3 +396,130 @@ async def test_build_agentic_loop_plan_missing_key_fallback():
plan.request_patch.messages[-1]["content"][0]["content"]
== "[compressed content key 'not_found.py' not found]"
)
def _stub_compress_result(original_tokens, compressed_tokens, cache):
return {
"messages": [{"role": "user", "content": "stubbed"}],
"original_tokens": original_tokens,
"compressed_tokens": compressed_tokens,
"compression_ratio": 0.5,
"cache": cache,
"tools": [],
}
@pytest.mark.asyncio
async def test_pre_call_hook_records_compression_savings_in_litellm_metadata(monkeypatch):
"""
When compression fires, the hook must record tokens_before/after/saved into
the request's litellm_metadata IN PLACE (the proxy and logging object hold
references to the same dict), so the savings land in the SpendLog row's
metadata JSON under ``compression_savings``.
"""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(12000, 5000, {"auth.py": "content"}),
)
litellm_metadata = {"user_api_key": "hashed-key", "user_api_key_user_id": "u1"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "very large context"}],
"litellm_metadata": litellm_metadata,
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert result["litellm_metadata"] is litellm_metadata
assert litellm_metadata["compression_savings"] == {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}
assert litellm_metadata["user_api_key"] == "hashed-key"
@pytest.mark.asyncio
async def test_pre_call_hook_creates_litellm_metadata_when_absent(monkeypatch):
"""SDK-direct calls have no litellm_metadata dict yet; the hook creates it."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(300, 100, {"k": "v"}),
)
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "ctx"}],
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert result["litellm_metadata"]["compression_savings"]["tokens_saved"] == 200
@pytest.mark.asyncio
@pytest.mark.parametrize(
"original_tokens,compressed_tokens",
[
(None, 5000),
(12000, None),
("12000", 5000),
(12000, "5000"),
(True, 5000),
(5000, 12000),
(12000, -1),
],
)
async def test_pre_call_hook_invalid_token_counts_fail_open(monkeypatch, original_tokens, compressed_tokens):
"""Invalid token counts must never crash the request; savings simply are not recorded."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: _stub_compress_result(original_tokens, compressed_tokens, {"k": "v"}),
)
litellm_metadata = {"user_api_key": "hashed-key"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "ctx"}],
"litellm_metadata": litellm_metadata,
}
result = await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert result is not None
assert "compression_savings" not in litellm_metadata
@pytest.mark.asyncio
async def test_pre_call_hook_no_compression_records_no_savings(monkeypatch):
"""When compression is a no-op (empty cache) nothing is recorded."""
logger = CompressionInterceptionLogger()
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: {
"messages": [],
"original_tokens": 100,
"compressed_tokens": 100,
"cache": {},
"tools": [],
"compression_skipped_reason": "below_trigger",
},
)
litellm_metadata = {"user_api_key": "hashed-key"}
kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "small"}],
"litellm_metadata": litellm_metadata,
}
await logger.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.anthropic_messages)
assert "compression_savings" not in litellm_metadata

41
tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py
View file

@ -206,6 +206,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"failed_requests": 0, # 0 + 0
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"compression_saved_tokens": 0,
}
updates = [{test_key: test_transaction1}, {test_key: test_transaction2}]
@ -253,6 +254,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"failed_requests": 0, # 0 + 0
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"compression_saved_tokens": 0,
}
# Add updates to queue
@ -476,3 +478,42 @@ async def test_queue_size_reduction_with_large_volume(
assert result[user2_key]["api_requests"] == 100
assert result[user2_key]["successful_requests"] == 100
assert result[user2_key]["failed_requests"] == 0
@pytest.mark.asyncio
async def test_compression_saved_tokens_aggregation(daily_spend_update_queue):
"""compression_saved_tokens must accumulate across payloads for the same key."""
test_key = "user1_2023-01-01_key123_claude-sonnet-5_anthropic"
transaction1 = {
"spend": 1.0,
"prompt_tokens": 10,
"completion_tokens": 5,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
"cache_read_input_tokens": 7,
"cache_creation_input_tokens": 3,
"compression_saved_tokens": 7000,
}
transaction2 = {
"spend": 2.0,
"prompt_tokens": 20,
"completion_tokens": 10,
"api_requests": 1,
"successful_requests": 1,
"failed_requests": 0,
"cache_read_input_tokens": 5,
"cache_creation_input_tokens": 4,
"compression_saved_tokens": 600,
}
await daily_spend_update_queue.add_update({test_key: transaction1})
await daily_spend_update_queue.add_update({test_key: transaction2})
await daily_spend_update_queue.aggregate_queue_updates()
updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue()
assert len(updates) == 1
agg = updates[0][test_key]
assert agg["compression_saved_tokens"] == 7600
assert agg["cache_read_input_tokens"] == 12
assert agg["cache_creation_input_tokens"] == 7

88
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -1902,3 +1902,91 @@ async def test_update_user_db_enqueues_user_spend_without_cache_dependency():
by_type = {u["entity_type"]: u["entity_id"] for u in queued}
assert by_type[Litellm_EntityType.USER] == "user-123"
assert by_type[Litellm_EntityType.END_USER] == "end-user-9"
@pytest.mark.asyncio
async def test_daily_transaction_carries_compression_saved_tokens():
"""
The daily transaction built from a SpendLog payload must carry
compression_saved_tokens summed from both the native compression_savings
metadata key and Headroom guardrail entries, alongside the cache token
fields extracted from usage_object.
"""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
mock_prisma.get_request_status = MagicMock(return_value="success")
metadata = {
"usage_object": {"cache_read_input_tokens": 40, "cache_creation_input_tokens": 15},
"compression_savings": {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
},
"guardrail_information": [
{
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {"tokens_before": 1000, "tokens_after": 400, "tokens_saved": 600},
}
],
}
payload = {
"request_id": "req-compression-1",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"call_type": "anthropic_messages",
"prompt_tokens": 5000,
"completion_tokens": 10,
"spend": 0.05,
"metadata": json.dumps(metadata),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload=payload,
prisma_client=mock_prisma,
type="user",
)
assert transaction is not None
assert transaction["compression_saved_tokens"] == 7600
assert transaction["cache_read_input_tokens"] == 40
assert transaction["cache_creation_input_tokens"] == 15
@pytest.mark.asyncio
async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
"""Requests without any compression metadata produce a zero count."""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
mock_prisma.get_request_status = MagicMock(return_value="success")
payload = {
"request_id": "req-no-compression",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
"model": "claude-sonnet-5",
"custom_llm_provider": "anthropic",
"model_group": "claude-sonnet-5",
"call_type": "anthropic_messages",
"prompt_tokens": 100,
"completion_tokens": 10,
"spend": 0.01,
"metadata": json.dumps({"usage_object": {}}),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload=payload,
prisma_client=mock_prisma,
type="user",
)
assert transaction is not None
assert transaction["compression_saved_tokens"] == 0

10
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -150,6 +150,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
"mcp_namespaced_tool_name": None,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"compression_saved_tokens": 0,
"failed_requests": 0,
}
mock_rows = [
@ -492,6 +493,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_1.completion_tokens = 200
mock_record_1.cache_read_input_tokens = 0
mock_record_1.cache_creation_input_tokens = 0
mock_record_1.compression_saved_tokens = 0
mock_record_1.api_requests = 10
mock_record_1.successful_requests = 9
mock_record_1.failed_requests = 1
@ -511,6 +513,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
mock_record_2.completion_tokens = 100
mock_record_2.cache_read_input_tokens = 0
mock_record_2.cache_creation_input_tokens = 0
mock_record_2.compression_saved_tokens = 0
mock_record_2.api_requests = 5
mock_record_2.successful_requests = 5
mock_record_2.failed_requests = 0
@ -570,6 +573,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
"mcp_namespaced_tool_name": None,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"compression_saved_tokens": 0,
"failed_requests": 0,
}
mock_rows = [
@ -654,6 +658,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend):
completion_tokens=5,
cache_read_input_tokens=0,
cache_creation_input_tokens=0,
compression_saved_tokens=0,
api_requests=1,
successful_requests=1,
failed_requests=0,
@ -865,6 +870,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
"completion_tokens": None,
"cache_read_input_tokens": None,
"cache_creation_input_tokens": None,
"compression_saved_tokens": None,
"api_requests": None,
"successful_requests": None,
"failed_requests": None,
@ -894,6 +900,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
assert result.metadata.total_failed_requests == 0
assert result.metadata.total_cache_read_input_tokens == 0
assert result.metadata.total_cache_creation_input_tokens == 0
assert result.metadata.total_compression_saved_tokens == 0
def _no_spend_record():
@ -904,6 +911,7 @@ def _no_spend_record():
completion_tokens=None,
cache_read_input_tokens=None,
cache_creation_input_tokens=None,
compression_saved_tokens=None,
api_requests=None,
successful_requests=None,
failed_requests=None,
@ -922,6 +930,7 @@ def test_record_to_spend_metrics_handles_none_values():
assert metrics.failed_requests == 0
assert metrics.cache_read_input_tokens == 0
assert metrics.cache_creation_input_tokens == 0
assert metrics.compression_saved_tokens == 0
def test_update_metrics_handles_none_values():
@ -936,3 +945,4 @@ def test_update_metrics_handles_none_values():
assert metrics.failed_requests == 0
assert metrics.cache_read_input_tokens == 0
assert metrics.cache_creation_input_tokens == 0
assert metrics.compression_saved_tokens == 0

110
tests/test_litellm/proxy/spend_tracking/test_compression_savings.py Normal file
View file

@ -0,0 +1,110 @@
"""
Unit tests for the compression-savings spend-log metadata normalizer.
"""
import pytest
from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
)
NATIVE_SAVINGS = {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}
HEADROOM_ENTRY = {
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {"tokens_before": 1000, "tokens_after": 400, "tokens_saved": 600},
}
def test_native_key_only():
assert extract_compression_saved_tokens({"compression_savings": NATIVE_SAVINGS}) == 7000
def test_headroom_only():
assert extract_compression_saved_tokens({"guardrail_information": [HEADROOM_ENTRY]}) == 600
def test_native_and_headroom_sum():
metadata = {
"compression_savings": NATIVE_SAVINGS,
"guardrail_information": [HEADROOM_ENTRY],
}
assert extract_compression_saved_tokens(metadata) == 7600
def test_multiple_headroom_entries_sum():
metadata = {"guardrail_information": [HEADROOM_ENTRY, HEADROOM_ENTRY]}
assert extract_compression_saved_tokens(metadata) == 1200
def test_neither_source_present():
assert extract_compression_saved_tokens({"user_api_key": "abc", "usage_object": {}}) == 0
def test_non_headroom_guardrail_entries_ignored():
metadata = {
"guardrail_information": [
{
"guardrail_name": "pii-guard",
"guardrail_provider": "presidio",
"guardrail_response": {"tokens_saved": 999},
}
]
}
assert extract_compression_saved_tokens(metadata) == 0
@pytest.mark.parametrize(
"compression_savings",
[
None,
"not-a-dict",
{},
{"tokens_saved": None},
{"tokens_saved": "7000"},
{"tokens_saved": 12.5},
{"tokens_saved": True},
{"tokens_saved": -5},
{"tokens_before": 100, "tokens_after": 50},
],
)
def test_malformed_native_key_contributes_zero(compression_savings):
assert extract_compression_saved_tokens({"compression_savings": compression_savings}) == 0
@pytest.mark.parametrize(
"guardrail_information",
[
None,
"not-a-list",
{"guardrail_provider": "headroom"},
[],
[None],
["not-a-dict"],
[{"guardrail_provider": "headroom"}],
[{"guardrail_provider": "headroom", "guardrail_response": "REDACTED"}],
[{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": "600"}}],
[{"guardrail_provider": "headroom", "guardrail_response": {"tokens_saved": -600}}],
[{"guardrail_response": {"tokens_saved": 600}}],
],
)
def test_malformed_headroom_information_contributes_zero(guardrail_information):
assert extract_compression_saved_tokens({"guardrail_information": guardrail_information}) == 0
def test_valid_headroom_entry_survives_alongside_malformed_ones():
metadata = {
"guardrail_information": [
None,
{"guardrail_provider": "headroom", "guardrail_response": "REDACTED"},
HEADROOM_ENTRY,
]
}
assert extract_compression_saved_tokens(metadata) == 600

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -1998,7 +1998,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -2094,7 +2094,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -2188,7 +2188,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

131
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -2571,3 +2571,134 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key():
assert not metadata_dict["user_api_key"].startswith("sk-"), (
f"metadata user_api_key contains unhashed key: {metadata_dict['user_api_key']}"
)
@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_preserves_headroom_compression_token_stats(
mock_should_store,
):
"""
Headroom's compression stats live in guardrail_response, which is redacted
by default. Purely numeric token stats carry no prompt content and must
survive so daily spend aggregation can count compression_saved_tokens;
everything else in guardrail_response stays redacted.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {
"tokens_before": 1000,
"tokens_after": 400,
"tokens_saved": 600,
"compression_ratio": 0.4,
"transforms_applied": ["dedupe_messages"],
},
},
{
"guardrail_name": "echo-guard",
"guardrail_status": "success",
"guardrail_response": {
"evaluated_input": "secret prompt",
"tokens_saved": "prompt text hiding in a stat key",
},
},
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
assert result[0]["guardrail_response"] == {
"tokens_before": 1000,
"tokens_after": 400,
"tokens_saved": 600,
"compression_ratio": 0.4,
}
assert result[1]["guardrail_response"] == REDACTED_BY_LITELM_STRING
@pytest.mark.asyncio
async def test_compression_savings_survive_to_spend_log_payload_metadata(monkeypatch):
"""
End-to-end seam test for the native compression write path on
/v1/messages: the pre-call deployment hook records savings into the call
kwargs' litellm_metadata, the logging object captures it via
update_from_kwargs (exactly as llm_http_handler does for
anthropic_messages), and get_logging_payload lands it in
SpendLogsPayload.metadata JSON under ``compression_savings``.
"""
from litellm.integrations.compression_interception.handler import (
CompressionInterceptionLogger,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import CallTypes
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: {
"messages": [{"role": "user", "content": "compressed"}],
"original_tokens": 12000,
"compressed_tokens": 5000,
"cache": {"auth.py": "content"},
"tools": [],
},
)
logger = CompressionInterceptionLogger()
call_kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "big context"}],
"max_tokens": 512,
"litellm_call_id": "test-compression-call-id",
"litellm_metadata": {
"user_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"user_api_key_user_id": "u1",
"user_api_key_team_id": "t1",
},
}
hooked = await logger.async_pre_call_deployment_hook(kwargs=call_kwargs, call_type=CallTypes.anthropic_messages)
assert hooked is not None
start_time = datetime.datetime.now(timezone.utc)
logging_obj = Logging(
model="claude-sonnet-5",
messages=hooked["messages"],
stream=False,
call_type="anthropic_messages",
start_time=start_time,
litellm_call_id="test-compression-call-id",
function_id="test",
)
logging_obj.update_from_kwargs(
kwargs=hooked,
model="claude-sonnet-5",
optional_params={"max_tokens": 512},
litellm_params={
"preset_cache_key": None,
"stream_response": {},
"model_info": hooked.get("model_info"),
},
custom_llm_provider="anthropic",
)
end_time = datetime.datetime.now(timezone.utc)
logging_obj.model_call_details["completion_start_time"] = end_time
payload = get_logging_payload(
kwargs=logging_obj.model_call_details,
response_obj={
"id": "msg_test",
"usage": {"prompt_tokens": 5000, "completion_tokens": 10, "total_tokens": 5010},
},
start_time=start_time,
end_time=end_time,
)
payload_metadata = json.loads(payload["metadata"])
assert payload_metadata["compression_savings"] == {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}

10
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -23239,6 +23239,11 @@ export interface components {
* @default 0
*/
total_completion_tokens: number;
/**
* Total Compression Saved Tokens
* @default 0
*/
total_compression_saved_tokens: number;
/**
* Total Failed Requests
* @default 0
@ -30864,6 +30869,11 @@ export interface components {
* @default 0
*/
completion_tokens: number;
/**
* Compression Saved Tokens
* @default 0
*/
compression_saved_tokens: number;
/**
* Failed Requests
* @default 0