diff --git a/litellm/accounting/snapshot.py b/litellm/accounting/snapshot.py index 07f64c90205..d5652f79ed0 100644 --- a/litellm/accounting/snapshot.py +++ b/litellm/accounting/snapshot.py @@ -4,10 +4,11 @@ from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter, ValidationError +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure from litellm.proxy._types import SpendLogsPayload from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens -from litellm.proxy.spend_tracking.savings import compute_savings_spend, marks_gateway_injection +from litellm.proxy.spend_tracking.savings import SavingsSpend, compute_savings_spend, marks_gateway_injection if TYPE_CHECKING: from litellm.router import Router @@ -66,20 +67,7 @@ def build_accounting_payload( router: "Callable[[], Router | None]", ) -> Mapping[str, object]: metadata: Final = _FIELDS.validate_json(payload["metadata"]) - savings: Final = compute_savings_spend( - model=payload["model"], - custom_llm_provider=payload["custom_llm_provider"], - compression_saved_tokens=extract_compression_saved_tokens(metadata), - gateway_injected_cache=marks_gateway_injection(metadata, payload["model_id"]), - routing_decision=_mapping(metadata.get("routing_decision")), - model_id=payload["model_id"], - llm_router=router, - usage_object=_mapping(metadata.get("usage_object")), - cost_breakdown=_mapping(metadata.get("cost_breakdown")), - recorded_autorouter_savings=metadata.get("autorouter_savings"), - recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")), - billed_at=payload["endTime"], - ) + savings: Final = _savings_or_zero(payload, metadata, router) accounting_metadata: Final = { **{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS}, "accounting_entities": entities, @@ -102,6 +90,39 @@ def build_accounting_payload( ) +def _savings_or_zero( + payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]" +) -> SavingsSpend: + """Savings are analytics computed from caller-supplied counts, so one that cannot be computed is stored as zero + rather than losing the charge it rides with.""" + try: + return _savings(payload, metadata, router) + except (ArithmeticError, ValueError, TypeError) as error: + verbose_proxy_logger.warning( + "Recording zero savings for request_id=%s, savings could not be computed: %s", payload["request_id"], error + ) + return SavingsSpend(compression=0.0, prompt_caching=0.0) + + +def _savings( + payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]" +) -> SavingsSpend: + return compute_savings_spend( + model=payload["model"], + custom_llm_provider=payload["custom_llm_provider"], + compression_saved_tokens=extract_compression_saved_tokens(metadata), + gateway_injected_cache=marks_gateway_injection(metadata, payload["model_id"]), + routing_decision=_mapping(metadata.get("routing_decision")), + model_id=payload["model_id"], + llm_router=router, + usage_object=_mapping(metadata.get("usage_object")), + cost_breakdown=_mapping(metadata.get("cost_breakdown")), + recorded_autorouter_savings=metadata.get("autorouter_savings"), + recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")), + billed_at=payload["endTime"], + ) + + _GUARDRAIL_ENTRIES: Final = TypeAdapter(tuple[dict[str, object], ...]) _GUARDRAIL_METADATA_FIELDS: Final = frozenset({"guardrail_information", "standard_logging_guardrail_information"}) _GUARDRAIL_ACCOUNTING_FIELDS: Final = frozenset( diff --git a/tests/unit/accounting/test_projection.py b/tests/unit/accounting/test_projection.py index 6ad466122c3..17186a89922 100644 --- a/tests/unit/accounting/test_projection.py +++ b/tests/unit/accounting/test_projection.py @@ -519,3 +519,39 @@ def test_stored_payload_keeps_guardrail_billing_but_not_guardrail_content() -> N assert [dict(item.increments) for item in usage_rows] == [{"units": 1, "cost": 0.25, "untracked_units": 0}] daily_user: Final = next(item for item in project(restored) if item.table == "LiteLLM_DailyUserSpend") assert dict(daily_user.increments)["compression_saved_tokens"] == 7 + + +def test_caller_supplied_savings_that_cannot_be_computed_never_drop_the_charge(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + + monkeypatch.setitem( + litellm.model_cost, + "savings-overflow-test", + {"input_cost_per_token": 0.002, "output_cost_per_token": 0.004, "litellm_provider": "openai", "mode": "chat"}, + ) + payload: Final[SpendLogsPayload] = { + **get_logging_payload( + kwargs={ + "model": "savings-overflow-test", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "overflow-request", + "litellm_params": {"metadata": {"user_api_key": "key"}}, + }, + response_obj=ModelResponse( + id="overflow-request", model="savings-overflow-test", usage=Usage(prompt_tokens=10, completion_tokens=5) + ), + start_time=_NOW, + end_time=_NOW, + ), + "spend": 0.5, + "metadata": json.dumps({"compression_savings": {"tokens_saved": 10**400}}), + } + accounting: Final = build_accounting_payload( + payload, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None + ) + restored: Final = _FIELDS.validate_json(json.dumps(accounting, default=str)) + assert dict(restored["metadata"])["accounting_savings"]["compression"] == 0.0 + assert [dict(item.increments)["spend"] for item in project(restored) if item.table == "LiteLLM_VerificationToken"] == [ + 0.5 + ]