From d6b1d23d45fc043e908fc8a21a41aabe5fbf44fd Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:27:16 -0700 Subject: [PATCH 01/20] feat(accounting): serialize durable cost compilation and replay --- litellm/proxy/db/accounting_ledger.py | 70 ++++++- litellm/proxy/spend_tracking/cost_compiler.py | 20 ++ .../database/test_accounting_ledger.py | 178 ++++++++++++++++++ 3 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/spend_tracking/cost_compiler.py diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 0a9c5db1619..37417244457 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -11,7 +11,17 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter from typing_extensions import assert_never -from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent +from litellm.accounting import ( + CostEvent, + DeliveryTarget, + EventKind, + InboundUsageEvent, + Metric, + MetricName, + OutboundUsageEvent, + PricingSnapshot, + Unpriced, +) from litellm.proxy.db.db_span import db_span from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper @@ -143,6 +153,14 @@ SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1 FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb) CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination """ +_LOAD_SOURCE: Final = """ +SELECT payload::text AS payload, schema_version::text AS schema_version +FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE +""" +_LOAD_COST: Final = """ +SELECT payload::text AS payload, schema_version::text AS schema_version, payload_hash +FROM "LiteLLM_AccountingEvent" WHERE source_event_id = $1 AND kind = 'cost' +""" _COMPLETED_FAMILY: Final = """ root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') AND root.recorded_at < $1::timestamp @@ -320,6 +338,56 @@ class AccountingLedger: case _: assert_never(result) + async def compile_cost( + self, + event: OutboundUsageEvent, + price: Callable[[str], PricingSnapshot | Unpriced], + targets: tuple[DeliveryTarget, ...], + ) -> CostEvent | Unpriced: + """Serialize pure pricing with other compilers and commit its complete delivery set.""" + validated: Final = OutboundUsageEvent.model_validate_json(event.model_dump_json()) + async with self._transaction() as db: + sources: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_SOURCE, validated.event_id)) + if not sources: + return Unpriced(usage_event_id=validated.event_id, reason="usage_not_committed") + source: Final = sources[0] + if source["schema_version"] != "1": + return Unpriced(usage_event_id=validated.event_id, reason="unsupported_usage_schema") + payload: Final = _JSON.validate_json(source["payload"]) + if payload.get("event") != validated.model_dump(mode="json"): + raise AccountingConflictError("Compilation input differs from the accepted usage event") + costs: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_COST, validated.event_id)) + if costs: + if costs[0]["schema_version"] != "1": + return Unpriced(usage_event_id=validated.event_id, reason="unsupported_cost_schema") + existing: Final = CostEvent.model_validate_json( + _json(_JSON.validate_json(costs[0]["payload"])["event"]) + ) + if existing.usage_event_id != validated.event_id: + raise AccountingConflictError("Committed cost does not match its source usage") + replay: Final = _prepare(existing, targets, costs[0]["payload"], 1, 1) + if replay.payload_hash != costs[0]["payload_hash"]: + raise AccountingConflictError("Compilation delivery set differs from the accepted cost") + return existing + priced: Final = price(source["payload"]) + if isinstance(priced, Unpriced): + if priced.usage_event_id != validated.event_id: + raise AccountingConflictError("Unpriced result does not match its source usage") + return priced + cost: Final = CostEvent( + event_id=f"cost:{validated.event_id}", + usage_event_id=validated.event_id, + occurred_at=validated.ended_at, + dimensions=validated.dimensions, + charges=(Metric(name=MetricName.SPEND, value=priced.total_cost, unit="USD"),), + snapshot=priced, + ) + record: Final = _prepare(cost, targets, _json({"event": cost.model_dump(mode="json")}), 1, 1) + accepted: Final = await _accept(db, record) + if accepted not in ("inserted", "duplicate"): + raise AccountingConflictError(f"Cannot commit compiled cost: {accepted}") + return cost + async def purge_completed( self, *, diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py new file mode 100644 index 00000000000..9d2438b93dc --- /dev/null +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -0,0 +1,20 @@ +from collections.abc import Callable +from typing import Final + +from litellm.accounting import CostEvent, DeliveryTarget, OutboundUsageEvent, PricingSnapshot, Unpriced +from litellm.proxy.db.accounting_ledger import AccountingLedger + + +class LoggingCostCompiler: + def __init__( + self, + ledger: AccountingLedger, + price_usage: Callable[[str], PricingSnapshot | Unpriced], + delivery_targets: tuple[DeliveryTarget, ...], + ) -> None: + self._ledger: Final = ledger + self._price_usage: Final = price_usage + self._delivery_targets: Final = delivery_targets + + async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced: + return await self._ledger.compile_cost(event, self._price_usage, self._delivery_targets) diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 15cd5413149..e107e8e130c 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -20,6 +20,7 @@ from psycopg import sql from pydantic import JsonValue from litellm.accounting import ( + CostCompiler, CostEvent, DeliveryState, DeliveryTarget, @@ -30,9 +31,11 @@ from litellm.accounting import ( OutboundUsageEvent, Outcome, PricingSnapshot, + Unpriced, ) from litellm.proxy.db import accounting_ledger from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite +from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler from litellm.types.utils import CallTypes from tests.integration._support.database import scratch_database from tests.integration._support.prisma_clients import connected_proxy_prisma @@ -248,6 +251,181 @@ async def test_event_and_complete_delivery_set_commit_together(writer: _Writer, assert set(deliveries) == {(target.value, "pending", 0) for target in _TARGETS} +@pytest.mark.asyncio +async def test_compilers_share_one_committed_charge_and_preserve_delivery_receipts( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event, context={"price": "0.125"}) + + def price(envelope: str) -> PricingSnapshot: + return _cost(amount=json.loads(envelope)["context"]["price"]).snapshot + + compiler: Final[CostCompiler] = LoggingCostCompiler(store, price, _TARGETS) + results: Final = await asyncio.gather(*(compiler.compile(event) for _ in range(8))) + assert all(result == results[0] for result in results) + assert isinstance(results[0], CostEvent) + assert results[0].snapshot.total_cost == Decimal("0.125") + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(1,)] + deliveries: Final = _rows( + ledger_database, + 'SELECT target::text, state::text FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s', + (results[0].event_id,), + ) + assert set(deliveries) == {(target.value, "pending") for target in _TARGETS} + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') + + def unavailable_pricing(envelope: str) -> PricingSnapshot: + raise AssertionError("Committed cost replay must not invoke pricing") + + replay: Final = await LoggingCostCompiler(store, unavailable_pricing, _TARGETS).compile(event) + assert replay == results[0] + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before + + +@pytest.mark.asyncio +async def test_compilation_rejects_changed_delivery_targets_without_resetting_receipts( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + original: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS[:1]).compile(event) + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') + with pytest.raises(AccountingConflictError, match="delivery set"): + await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before + assert await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS[:1]).compile(event) == original + + +@pytest.mark.asyncio +async def test_compilation_preserves_unknown_cost_without_creating_a_zero_charge( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + unresolved: Final = Unpriced(usage_event_id=event.event_id, reason="unknown_usage") + compiler: Final = LoggingCostCompiler(store, lambda _: unresolved, _TARGETS) + assert await compiler.compile(event) == Unpriced(usage_event_id=event.event_id, reason="usage_not_committed") + await _insert(store, event) + assert await compiler.compile(event) == unresolved + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + + +@pytest.mark.asyncio +async def test_compilation_rejects_an_unpriced_result_for_another_source(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + compiler: Final = LoggingCostCompiler( + store, lambda _: Unpriced(usage_event_id="other", reason="missing_price"), _TARGETS + ) + with pytest.raises(AccountingConflictError, match="Unpriced result"): + await compiler.compile(event) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + + +@pytest.mark.asyncio +async def test_compilation_rejects_a_cost_identity_collision(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + await _insert(store, _usage(f"cost:{event.event_id}", inbound=True)) + with pytest.raises(AccountingConflictError, match="identity_conflict"): + await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + + +@pytest.mark.asyncio +async def test_compilation_rejects_changed_source_before_reusing_a_charge(writer: _Writer) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + compiler: Final = LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS) + assert isinstance(await compiler.compile(event), CostEvent) + changed: Final = event.model_copy(update={"outcome": Outcome.CANCELLED}) + with pytest.raises(AccountingConflictError, match="differs from the accepted usage"): + await compiler.compile(changed) + + +@pytest.mark.asyncio +async def test_compilation_failure_rolls_back_and_can_be_retried(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + + def failed_pricing(envelope: str) -> PricingSnapshot: + raise ValueError("malformed retained pricing") + + with pytest.raises(ValueError, match="malformed retained pricing"): + await LoggingCostCompiler(store, failed_pricing, _TARGETS).compile(event) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + result: Final = await LoggingCostCompiler(store, lambda _: _cost(amount="0").snapshot, _TARGETS).compile(event) + assert isinstance(result, CostEvent) + assert result.charges[0].value == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["usage", "cost"]) +async def test_compilation_does_not_interpret_unknown_envelope_versions(writer: _Writer, kind: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await store.insert_event(event, _TARGETS, envelope=_envelope(event), schema_version=2 if kind == "usage" else 1) + if kind == "cost": + cost: Final = _cost() + await store.insert_event(cost, _TARGETS, envelope=_envelope(cost), schema_version=2) + result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert result == Unpriced(usage_event_id=event.event_id, reason=f"unsupported_{kind}_schema") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "cancellation"]) +async def test_compiler_recovery_preserves_atomic_charge_and_delivery_acceptance( + writer: _Writer, ledger_database: str, failure: str +) -> None: + client, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + + @asynccontextmanager + async def interrupted() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + if failure == "before_commit": + raise RuntimeError("interrupted cost commit") + if failure == "cancellation": + raise asyncio.CancelledError + raise RuntimeError("lost cost acknowledgement") + + compiler: Final = LoggingCostCompiler(AccountingLedger(interrupted), lambda _: _cost().snapshot, _TARGETS) + error: Final = asyncio.CancelledError if failure == "cancellation" else RuntimeError + with pytest.raises(error): + await compiler.compile(event) + count: Final = int(failure == "after_commit") + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(count,)] + assert _rows( + ledger_database, + 'SELECT count(*) FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s', + (f"cost:{event.event_id}",), + ) == [(count * len(_TARGETS),)] + result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert isinstance(result, CostEvent) + assert result.snapshot == _cost().snapshot + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(1,)] + + @pytest.mark.asyncio async def test_concurrent_duplicates_preserve_completed_deliveries(writer: _Writer, ledger_database: str) -> None: _, store = writer From 26c6c23c48362d996ea7beeed1895757215fe9f2 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 08:31:00 -0700 Subject: [PATCH 02/20] feat(accounting): compile retained usage into authoritative costs --- .../migration.sql | 5 + .../litellm_proxy_extras/schema.prisma | 3 + litellm/cost_calculator.py | 97 +- litellm/litellm_core_utils/litellm_logging.py | 126 ++- .../llm_cost_calc/pricing_context.py | 65 ++ litellm/main.py | 55 +- litellm/proxy/db/accounting_ledger.py | 44 +- litellm/proxy/proxy_server.py | 22 +- litellm/proxy/schema.prisma | 3 + litellm/proxy/spend_tracking/cost_compiler.py | 324 ++++++- litellm/proxy/spend_tracking/usage_capture.py | 112 ++- litellm/utils.py | 5 + schema.prisma | 3 + .../database/test_accounting_ledger.py | 210 ++++- .../spend_tracking/test_cost_compiler.py | 849 ++++++++++++++++++ .../spend_tracking/test_usage_capture.py | 74 +- tests/unit/test_cost_calculator.py | 47 + .../test_register_model_custom_pricing.py | 370 ++++---- 18 files changed, 2132 insertions(+), 282 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql create mode 100644 litellm/litellm_core_utils/llm_cost_calc/pricing_context.py create mode 100644 tests/unit/proxy/spend_tracking/test_cost_compiler.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql new file mode 100644 index 00000000000..04fbdfdc237 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql @@ -0,0 +1,5 @@ +ALTER TABLE "LiteLLM_AccountingEvent" +ADD COLUMN "compilation_state" TEXT NOT NULL DEFAULT 'pending', +ADD COLUMN "compilation_error" TEXT; +CREATE INDEX "LiteLLM_AccountingEvent_kind_compilation_state_recorded_at_idx" +ON "LiteLLM_AccountingEvent"("kind", "compilation_state", "recorded_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 179b741fe7c..048e0e1e17a 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2111,9 +2111,12 @@ model LiteLLM_AccountingEvent { occurred_at DateTime recorded_at DateTime @default(now()) payload Json + compilation_state String @default("pending") + compilation_error String? @@index([recorded_at]) @@index([kind, source_event_id]) + @@index([kind, compilation_state, recorded_at]) } // One row per (event, target table, version): pending work for the projector and, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e66be233be4..76cced0c1c9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -26,6 +26,12 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.llm_cost_calc.pricing_context import ( + pricing_discounts, + pricing_margins, + retained_model_info, + retained_models, +) from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -379,7 +385,9 @@ def _per_second_pricing_cost( audio_seconds: float = 0.0, ) -> tuple[float, float] | None: try: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): @@ -538,7 +546,7 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] - model_cost_ref: Final = litellm.model_cost + model_cost_ref: Final = retained_models() or litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy # behavior: `model_with_provider` stays equal to the raw `model` string @@ -600,9 +608,11 @@ def cost_per_token( model = model_without_prefix if is_decisions_call_type(call_type): - decisions_model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + decisions_model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) decisions_overrides: Final = decisions_pricing_fields( - litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded + (retained_models() or litellm.model_cost).get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) if decisions_overrides: return generic_cost_per_token( @@ -623,7 +633,9 @@ def cost_per_token( ) if lyria_generation_cost is not None: return 0.0, lyria_generation_cost - speech_model_info: Final = cached_get_model_info_helper( + speech_model_info: Final = retained_model_info( + model_without_prefix, custom_llm_provider + ) or cached_get_model_info_helper( model=model_without_prefix, custom_llm_provider=custom_llm_provider, preserve_unset_costs=True, @@ -808,7 +820,9 @@ def cost_per_token( service_tier=service_tier, ) else: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, @@ -995,8 +1009,8 @@ def select_model_name_for_cost_calc( ) if custom_pricing is True: - if router_model_id is not None and router_model_id in litellm.model_cost: - entry: Final = litellm.model_cost[router_model_id] + if router_model_id is not None and router_model_id in (retained_models() or litellm.model_cost): + entry: Final = (retained_models() or litellm.model_cost)[router_model_id] if _cost_map_entry_prices_anything(entry): return_model = router_model_id else: @@ -1075,14 +1089,16 @@ def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069).""" segments: Final = model.split("/") - if "/".join(segments[1:]) in litellm.model_cost: + if "/".join(segments[1:]) in (retained_models() or litellm.model_cost): return model head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1 head: Final = "/".join(segments[:head_len]) tail: Final = segments[head_len:] strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail)) candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1)) - return next((candidate for candidate in candidates if candidate in litellm.model_cost), model) + return next( + (candidate for candidate in candidates if candidate in (retained_models() or litellm.model_cost)), model + ) @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) @@ -1348,9 +1364,7 @@ def _apply_cost_discount( Tuple of (final_cost, discount_percent, discount_amount) """ original_cost: Final = base_cost - resolved_discount: Final = resolve_cost_discount( - litellm.cost_discount_config, custom_llm_provider, model, region_name - ) + resolved_discount: Final = resolve_cost_discount(pricing_discounts(), custom_llm_provider, model, region_name) discount_percent: Final = 0.0 if resolved_discount is None else resolved_discount discount_amount: Final = original_cost * discount_percent @@ -1389,12 +1403,12 @@ def _apply_cost_margin( # Get margin config - check provider-specific first, then global margin_config = None - if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config: - margin_config = litellm.cost_margin_config[custom_llm_provider] + if custom_llm_provider and custom_llm_provider in pricing_margins(): + margin_config = pricing_margins()[custom_llm_provider] if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug("Found provider-specific margin config for %s: %s", custom_llm_provider, margin_config) - elif "global" in litellm.cost_margin_config: - margin_config = litellm.cost_margin_config["global"] + elif "global" in pricing_margins(): + margin_config = pricing_margins()["global"] if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug("Using global margin config: %s", margin_config) else: @@ -1402,7 +1416,7 @@ def _apply_cost_margin( verbose_logger.debug( "No margin config found. Provider: %s, Available configs: %s", custom_llm_provider, - list(litellm.cost_margin_config.keys()), + list(pricing_margins().keys()), ) if margin_config is not None: @@ -2159,12 +2173,14 @@ def completion_cost( # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" - ) and not has_together_registry_pricing(model, litellm.model_cost): + ) and not has_together_registry_pricing(model, (retained_models() or litellm.model_cost)): model = get_model_params_and_category(model, call_type=CallTypes(call_type)) # replicate llms are calculate based on time for request running # see https://replicate.com/pricing - elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: + elif (model in litellm.replicate_models or "replicate" in model) and model not in ( + retained_models() or litellm.model_cost + ): # for unmapped replicate model, default to replicate's time tracking logic return get_replicate_completion_pricing(completion_response, response_time_ms) @@ -2261,7 +2277,7 @@ def completion_cost( _final_cost += sum(additional_costs.values()) original_cost = _final_cost - if litellm.cost_discount_config: + if pricing_discounts(): ( _final_cost, discount_percent, @@ -2277,7 +2293,7 @@ def completion_cost( discount_amount = 0.0 # Apply margin from module-level config if configured - if litellm.cost_margin_config: + if pricing_margins(): ( _final_cost, margin_percent, @@ -2629,7 +2645,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, . def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: - raw_entry: Final = litellm.model_cost.get(key) + raw_entry: Final = (retained_models() or litellm.model_cost).get(key) return raw_entry if isinstance(raw_entry, Mapping) else None @@ -2973,12 +2989,19 @@ def default_image_cost_calculator( model_name_without_custom_llm_provider, ) matched_model: Final = next( - (_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None + ( + _model + for _model in models_to_check + if _model is not None and _model in (retained_models() or litellm.model_cost) + ), + None, ) if matched_model is None and model_info is None: raise Exception(f"Model not found in cost map. Tried checking {models_to_check}") - shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None + shared_cost_info: Final = ( + (retained_models() or litellm.model_cost)[matched_model] if matched_model is not None else None + ) price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None) image_count: Final = n if n is not None else 1 unit_counts: Final = ( @@ -3016,7 +3039,7 @@ def default_video_cost_calculator( custom_llm_provider (Optional[str]): Custom LLM provider model_info (Optional[ModelInfo]): Deployment-level model info containing custom video pricing. When provided, used before falling back to - the global litellm.model_cost lookup. + the effective model cost lookup. video_resolution (Optional[str]): From usage (e.g. ``720p``, ``1080p``) for tiered per-second pricing. Returns: @@ -3026,9 +3049,9 @@ def default_video_cost_calculator( Exception: If model pricing not found in cost map """ # Use custom model_info pricing if provided (deployment-specific pricing) - cost_info: dict | None = None + cost_info: ModelInfo | None = None if model_info is not None: - cost_info = dict(model_info) + cost_info = model_info else: # Build model names for cost lookup base_model_name = model @@ -3049,15 +3072,15 @@ def default_video_cost_calculator( model_name_without_custom_llm_provider, ] for _model in models_to_check: - if _model is not None and _model in litellm.model_cost: - cost_info = litellm.model_cost[_model] + if _model is not None and _model in (retained_models() or litellm.model_cost): + cost_info = (retained_models() or litellm.model_cost)[_model] break # If still not found, try with custom_llm_provider prefix if cost_info is None and custom_llm_provider: prefixed_model: Final = f"{custom_llm_provider}/{model}" - if prefixed_model in litellm.model_cost: - cost_info = litellm.model_cost[prefixed_model] + if prefixed_model in (retained_models() or litellm.model_cost): + cost_info = (retained_models() or litellm.model_cost)[prefixed_model] if cost_info is None: raise Exception(f"Model not found in cost map for model={model}") @@ -3663,9 +3686,9 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) """ resolved: Final = _cost_map_model_info(model_name, custom_llm_provider) entries: Final = ( - litellm.model_cost.get(resolved.get("key")) if resolved is not None else None, - litellm.model_cost.get(model_name), - litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"), + (retained_models() or litellm.model_cost).get(resolved.get("key")) if resolved is not None else None, + (retained_models() or litellm.model_cost).get(model_name), + (retained_models() or litellm.model_cost).get(f"{custom_llm_provider}/{model_name}"), ) return any(entry is not None and _cost_map_entry_prices_anything(entry) for entry in entries) @@ -3910,11 +3933,11 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: - """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry + """First of ``keys`` this entry prices, read off the raw model cost entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: return None - declared: Final = litellm.model_cost.get(info.get("key")) + declared: Final = (retained_models() or litellm.model_cost).get(info.get("key")) if declared is None: return None return next( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 639003c9a6a..6d9df8a7ad5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -14,6 +14,7 @@ import sys import time import traceback from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime as dt_object from enum import Enum @@ -25,7 +26,13 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase +from litellm._internal_context import ( + current_billing_time, + in_mirrored_call, + outside_mirrored_call, + pinned_billing_time, + post_response_phase, +) from litellm._logging import ( is_debugging_on, redact_string, @@ -112,6 +119,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( object_mapping, with_guardrail_entry, ) +from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -666,6 +674,10 @@ class UsageCaptureObservation: return cls(serialized, provenance, response_cost) +_REQUEST_MODEL_INFO: Final = TypeAdapter(ModelInfo) +_REQUEST_PRICING_MODELS: Final = TypeAdapter(dict[str, ModelInfo]) + + class Logging(LiteLLMLoggingBaseClass): usage_capture_callback: ClassVar[ Callable[ @@ -703,6 +715,8 @@ class Logging(LiteLLMLoggingBaseClass): logfireLogger, \ prometheusLogger, \ slack_app + cost_billing_time: datetime.datetime | None = None + request_pricing: RetainedPricing | None = None custom_pricing: bool = False custom_cost_per_token: CostPerToken | None = None custom_cost_per_second: float | None = None @@ -2107,6 +2121,57 @@ class Logging(LiteLLMLoggingBaseClass): if margin_total_amount is not None: self.cost_breakdown["margin_total_amount"] = margin_total_amount + def _request_pricing_model_info( + self, model: str, provider: str, overrides: Mapping[str, object], deployment_id: str | None + ) -> ModelInfo: + for candidate in (deployment_id, f"{provider}/{model}"): + if candidate is None: + continue + try: + return litellm.get_model_info(model=candidate, custom_llm_provider=provider) + except Exception: # noqa: BLE001 # unmapped SDK models can supply their complete pricing + verbose_logger.debug("No base pricing for request override candidate %s", candidate) + return _REQUEST_MODEL_INFO.validate_python( + { + "key": deployment_id or f"{provider}/{model}", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": provider, + "mode": overrides.get( + "mode", "embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat" + ), + "supported_openai_params": None, + } + ) + + def set_request_pricing( + self, + model: str, + provider: str, + overrides: Mapping[str, object], + deployment_id: str | None = None, + ) -> None: + from litellm.utils import model_info_with_pricing_overrides + + base: Final = self._request_pricing_model_info(model, provider, overrides, deployment_id) + normalized: Final = _REQUEST_MODEL_INFO.validate_python( + { + **base, + "mode": base.get("mode") + or ("embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat"), + } + ) + effective: Final = model_info_with_pricing_overrides(normalized, overrides) + keys: Final = (model, f"{provider}/{model}", deployment_id or model, effective["key"]) + self.request_pricing = RetainedPricing( + models_json=_REQUEST_PRICING_MODELS.dump_json({name: effective for name in keys}).decode(), + discounts_json=json.dumps(litellm.cost_discount_config), + margins_json=json.dumps(litellm.cost_margin_config), + ) + def response_cost_calculator( self, result: object, @@ -2114,6 +2179,24 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name: str | None = None, router_model_id: str | None = None, ignore_stamped_cost: bool = False, + ) -> float | None: + context: Final = ( + retained_pricing(self.request_pricing) + if self.request_pricing is not None and not self.spilled_over_ptu_for(result) + else nullcontext() + ) + with context, pinned_billing_time(self.billing_time_for_cost()): + return self._calculate_response_cost( + result, cache_hit, litellm_model_name, router_model_id, ignore_stamped_cost + ) + + def _calculate_response_cost( + self, + result: object, + cache_hit: bool | None = None, + litellm_model_name: str | None = None, + router_model_id: str | None = None, + ignore_stamped_cost: bool = False, ) -> float | None: """ Calculate response cost using result + logging object variables. @@ -2404,7 +2487,9 @@ class Logging(LiteLLMLoggingBaseClass): def custom_pricing_for(self, result: object) -> bool: if self.spilled_over_ptu_for(result): return False - return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None)) + return self.request_pricing is not None or use_custom_pricing_for_model( + litellm_params=getattr(self, "litellm_params", None) + ) def _prompt_for_cost_calculation(self) -> str: """ @@ -2965,6 +3050,8 @@ class Logging(LiteLLMLoggingBaseClass): metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY) return { "id": self.litellm_call_id, + "model": self.model, + "results": result, "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")} if metered is not None else None, @@ -2974,14 +3061,29 @@ class Logging(LiteLLMLoggingBaseClass): ) return normalized if assembled is None else assembled + def record_provider_reported_cost(self, cost: float | None) -> None: + self._provider_reported_cost = cost + + def billing_time_for_cost(self) -> datetime.datetime: + if self.cost_billing_time is None: + self.cost_billing_time = current_billing_time() + return self.cost_billing_time + def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]: from litellm.utils import count_characters return TypeAdapter(dict[str, JsonValue]).validate_python( { + "response_ms": getattr(result, "_response_ms", None), + "provider_reported_cost": self._provider_reported_cost, + "current_billing_time": self.billing_time_for_cost().isoformat(), "custom_pricing": self.custom_pricing_for(result), "custom_cost_per_token": self.custom_cost_per_token, "custom_cost_per_second": self.custom_cost_per_second, + "tool_response_cost": self.model_call_details.get("response_cost") + if self.model_call_details.get("call_type") + in (CallTypes.call_mcp_tool.value, CallTypes.send_message.value, CallTypes.asend_message.value) + else None, "prompt_characters": count_characters(self._prompt_for_cost_calculation()) if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value) else None, @@ -7838,14 +7940,20 @@ def get_standard_logging_object_payload( timing_metrics: Final = getattr(logging_obj, "response_timing_metrics", None) or {} clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms") - model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information( - base_model=base_model, - custom_pricing=custom_pricing, - custom_llm_provider=kwargs.get("custom_llm_provider"), - init_response_obj=init_response_obj, - api_base=litellm_params.get("api_base"), - router_model_id=logging_obj.get_router_model_id(), + pricing_context: Final = ( + retained_pricing(logging_obj.request_pricing) + if logging_obj.request_pricing is not None and not spilled_over + else nullcontext() ) + with pricing_context: + model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=base_model, + custom_pricing=custom_pricing, + custom_llm_provider=kwargs.get("custom_llm_provider"), + init_response_obj=init_response_obj, + api_base=litellm_params.get("api_base"), + router_model_id=logging_obj.get_router_model_id(), + ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( metadata=metadata, diff --git a/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py new file mode 100644 index 00000000000..f9509cac41e --- /dev/null +++ b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py @@ -0,0 +1,65 @@ +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Final + +from pydantic import TypeAdapter + +from litellm.types.utils import ModelInfo + +_MODELS: Final = TypeAdapter(dict[str, ModelInfo]) +_DISCOUNTS: Final = TypeAdapter(dict[str, float]) +_MARGINS: Final = TypeAdapter(dict[str, float | dict[str, float]]) + + +@dataclass(frozen=True, slots=True) +class RetainedPricing: + models_json: str + discounts_json: str + margins_json: str + + def models(self) -> Mapping[str, ModelInfo]: + return _MODELS.validate_json(self.models_json) + + +_CURRENT: Final[ContextVar[RetainedPricing | None]] = ContextVar("retained_pricing", default=None) + + +@contextmanager +def retained_pricing(pricing: RetainedPricing) -> Generator[None]: + token: Final = _CURRENT.set(pricing) + try: + yield + finally: + _CURRENT.reset(token) + + +def retained_model_info(model: str, provider: str | None) -> ModelInfo | None: + context: Final = _CURRENT.get() + if context is None: + return None + models: Final = context.models() + result: Final = models.get(f"{provider}/{model}") or models.get(model) + if result is None: + raise ValueError(f"Pricing snapshot has no retained model {model}") + return result + + +def retained_models() -> Mapping[str, ModelInfo] | None: + context: Final = _CURRENT.get() + return None if context is None else context.models() + + +def pricing_discounts() -> Mapping[str, float]: + import litellm + + context: Final = _CURRENT.get() + return litellm.cost_discount_config if context is None else _DISCOUNTS.validate_json(context.discounts_json) + + +def pricing_margins() -> Mapping[str, float | Mapping[str, float]]: + import litellm + + context: Final = _CURRENT.get() + return litellm.cost_margin_config if context is None else _MARGINS.validate_json(context.margins_json) diff --git a/litellm/main.py b/litellm/main.py index 78767b03eb7..170a4cd3551 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1316,7 +1316,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) from litellm.responses.bridge_check import ( bridge_reasoning_effort, - keep_registered_mode, str_keyed_mapping, ) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site @@ -1472,50 +1471,20 @@ def _record_agentic_loop_params( ) -def _register_custom_pricing_for_request( +def _set_request_pricing( + logging_obj: object, model: str, custom_llm_provider: str, kwargs: dict, model_info: dict | None, ) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) + if not isinstance(logging_obj, LiteLLMLoggingObj): return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) - ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, + logging_obj.set_request_pricing( + model, + custom_llm_provider, + _build_custom_pricing_entry(custom_llm_provider, kwargs, model_info), + _get_router_deployment_id(kwargs), ) @@ -5973,14 +5942,14 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### if ( (input_cost_per_token is not None and output_cost_per_token is not None) or input_cost_per_second is not None or output_cost_per_second is not None or cost_per_second is not None ): - _register_custom_pricing_for_request( + _set_request_pricing( + logging_obj=litellm_logging_obj, model=model, custom_llm_provider=custom_llm_provider, kwargs=kwargs, @@ -7044,14 +7013,14 @@ def embedding( **non_default_params, ) - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### if ( (input_cost_per_token is not None and output_cost_per_token is not None) or input_cost_per_second is not None or output_cost_per_second is not None or cost_per_second is not None ): - _register_custom_pricing_for_request( + _set_request_pricing( + logging_obj=litellm_logging_obj, model=model, custom_llm_provider=custom_llm_provider, kwargs=kwargs, diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 37417244457..b479cea6a0d 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -161,6 +161,24 @@ _LOAD_COST: Final = """ SELECT payload::text AS payload, schema_version::text AS schema_version, payload_hash FROM "LiteLLM_AccountingEvent" WHERE source_event_id = $1 AND kind = 'cost' """ +_PENDING_COMPILATION: Final = """ +SELECT payload::text AS payload FROM "LiteLLM_AccountingEvent" +WHERE kind = 'outbound_usage' AND compilation_state = 'pending' +ORDER BY recorded_at, event_id LIMIT $1::int +""" +_COMPILATION_RESULT: Final = """ +UPDATE "LiteLLM_AccountingEvent" SET compilation_state = $2, compilation_error = $3 +WHERE event_id = $1 AND kind = 'outbound_usage' +""" + + +async def _unpriced(db: Prisma, event_id: str, reason: str) -> Unpriced: + await db.execute_raw( + _COMPILATION_RESULT, event_id, "excluded" if reason == "not_charge_owner" else "unpriced", reason + ) + return Unpriced(usage_event_id=event_id, reason=reason) + + _COMPLETED_FAMILY: Final = """ root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') AND root.recorded_at < $1::timestamp @@ -338,6 +356,15 @@ class AccountingLedger: case _: assert_never(result) + async def pending_compilation(self, limit: int = 100) -> tuple[OutboundUsageEvent, ...]: + if not 1 <= limit <= 1000: + raise ValueError("Compilation batch size must be between 1 and 1000") + async with self._transaction() as db: + rows: Final = _STRINGS.validate_python(await db.query_raw(_PENDING_COMPILATION, limit)) + return tuple( + OutboundUsageEvent.model_validate_json(_json(_JSON.validate_json(row["payload"])["event"])) for row in rows + ) + async def compile_cost( self, event: OutboundUsageEvent, @@ -352,14 +379,14 @@ class AccountingLedger: return Unpriced(usage_event_id=validated.event_id, reason="usage_not_committed") source: Final = sources[0] if source["schema_version"] != "1": - return Unpriced(usage_event_id=validated.event_id, reason="unsupported_usage_schema") + return await _unpriced(db, validated.event_id, "unsupported_usage_schema") payload: Final = _JSON.validate_json(source["payload"]) if payload.get("event") != validated.model_dump(mode="json"): raise AccountingConflictError("Compilation input differs from the accepted usage event") costs: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_COST, validated.event_id)) if costs: if costs[0]["schema_version"] != "1": - return Unpriced(usage_event_id=validated.event_id, reason="unsupported_cost_schema") + return await _unpriced(db, validated.event_id, "unsupported_cost_schema") existing: Final = CostEvent.model_validate_json( _json(_JSON.validate_json(costs[0]["payload"])["event"]) ) @@ -368,12 +395,16 @@ class AccountingLedger: replay: Final = _prepare(existing, targets, costs[0]["payload"], 1, 1) if replay.payload_hash != costs[0]["payload_hash"]: raise AccountingConflictError("Compilation delivery set differs from the accepted cost") + await db.execute_raw(_COMPILATION_RESULT, validated.event_id, "compiled", None) return existing - priced: Final = price(source["payload"]) + try: + priced: Final = price(source["payload"]) + except ValueError: + return await _unpriced(db, validated.event_id, "invalid_pricing_inputs") if isinstance(priced, Unpriced): if priced.usage_event_id != validated.event_id: raise AccountingConflictError("Unpriced result does not match its source usage") - return priced + return await _unpriced(db, validated.event_id, priced.reason) cost: Final = CostEvent( event_id=f"cost:{validated.event_id}", usage_event_id=validated.event_id, @@ -382,10 +413,13 @@ class AccountingLedger: charges=(Metric(name=MetricName.SPEND, value=priced.total_cost, unit="USD"),), snapshot=priced, ) - record: Final = _prepare(cost, targets, _json({"event": cost.model_dump(mode="json")}), 1, 1) + record: Final = _prepare( + cost, targets, _json({"event": cost.model_dump(mode="json"), "pricing_inputs": payload}), 1, 1 + ) accepted: Final = await _accept(db, record) if accepted not in ("inserted", "duplicate"): raise AccountingConflictError(f"Cannot commit compiled cost: {accepted}") + await db.execute_raw(_COMPILATION_RESULT, validated.event_id, "compiled", None) return cost async def purge_completed( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b5f4a36f138..7980069b0ff 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3228,11 +3228,15 @@ def _configure_usage_capture() -> None: "Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier" ) from litellm.proxy.db.accounting_ledger import AccountingLedger + from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler ledger: Final = AccountingLedger.for_client(prisma_client) writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch) usage_capture.start( - writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch + writer.write, + UsageCaptureSettings.model_validate(capture_settings), + load=ledger.load_usage_batch, + compiler=LoggingCostCompiler(ledger), ) @@ -11591,6 +11595,22 @@ class ProxyStartupEvent: await cls._initialize_spend_tracking_background_jobs(scheduler=scheduler) + async def compile_accounting_costs() -> None: + from litellm.proxy.db.accounting_ledger import AccountingLedger + from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler + + if usage_capture.meter is not None: + await LoggingCostCompiler(AccountingLedger.for_client(prisma_client)).compile_pending() + + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler has no typed add_job interface + compile_accounting_costs, + "interval", + seconds=5, + id="accounting_cost_compilation", + replace_existing=True, + max_instances=1, + ) + cls._initialize_spend_capture_rate_check_job( scheduler=scheduler, proxy_logging_obj=proxy_logging_obj, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 179b741fe7c..048e0e1e17a 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2111,9 +2111,12 @@ model LiteLLM_AccountingEvent { occurred_at DateTime recorded_at DateTime @default(now()) payload Json + compilation_state String @default("pending") + compilation_error String? @@index([recorded_at]) @@index([kind, source_event_id]) + @@index([kind, compilation_state, recorded_at]) } // One row per (event, target table, version): pending work for the projector and, diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 9d2438b93dc..3580d2854e2 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -1,20 +1,334 @@ -from collections.abc import Callable +import json +import math +from collections.abc import Callable, Mapping +from datetime import datetime +from decimal import Decimal from typing import Final -from litellm.accounting import CostEvent, DeliveryTarget, OutboundUsageEvent, PricingSnapshot, Unpriced +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm._internal_context import pinned_billing_time +from litellm._logging import verbose_proxy_logger +from litellm.accounting import CostEvent, DeliveryTarget, DimensionName, OutboundUsageEvent, PricingSnapshot, Unpriced +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost +from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing from litellm.proxy.db.accounting_ledger import AccountingLedger +from litellm.proxy.spend_tracking.usage_meter import usage_delivery_targets +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import ( + CallTypes, + CostPerToken, + LiteLLMRealtimeStreamLoggingObject, + ModelResponse, + StandardBuiltInToolsParams, + Usage, + is_server_derived_pricing_key, +) + +_FIELDS: Final = TypeAdapter(dict[str, JsonValue]) + + +def _fields(value: object) -> Mapping[str, JsonValue]: + return _FIELDS.validate_python(value) if isinstance(value, dict) else {} + + +def _json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +class _UsageEnvelope(BaseModel): + model_config = ConfigDict(frozen=True) + + event: OutboundUsageEvent + usage: Mapping[str, JsonValue] | None = None + pricing: Mapping[str, JsonValue] + charge_eligible: bool + finalized_cost: Mapping[str, JsonValue] | None = None + allocation: Mapping[str, JsonValue] | None = None + + +def price_usage(envelope: str) -> PricingSnapshot | Unpriced: + captured: Final = _UsageEnvelope.model_validate_json(envelope) + event: Final = captured.event + if not captured.charge_eligible: + return Unpriced(usage_event_id=event.event_id, reason="not_charge_owner") + fixed_call: Final = event.call_type in {CallTypes.call_mcp_tool, CallTypes.send_message, CallTypes.asend_message} + realtime_events: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python( + captured.pricing.get("realtime_events") or () + ) + realtime_usage: Final = event.call_type is CallTypes.arealtime and any( + bool(item.get("usage")) for item in realtime_events + ) + if captured.usage is None and not fixed_call and not realtime_usage: + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") + pricing: Final = captured.pricing + mapped: Final = _fields(pricing.get("model_map_information")) + rates: Final = _fields(mapped.get("model_map_value")) + model: Final = next( + (d.value for d in event.dimensions if d.name is DimensionName.MODEL), + event.call_type.value if fixed_call else None, + ) + provider: Final = next((d.value for d in event.dimensions if d.name is DimensionName.PROVIDER), None) + logged: Final = _fields(pricing.get("logging")) + reported: Final = logged.get("provider_reported_cost") + upstream: Final = ( + isinstance(reported, (int, float)) + and not isinstance(reported, bool) + and math.isfinite(reported) + and reported >= 0 + and logged.get("custom_pricing") is not True + ) + key: Final = mapped.get("model_map_key") or (model if fixed_call or upstream else None) + has_rates: Final = any(is_server_derived_pricing_key(name) and value is not None for name, value in rates.items()) + if model is None or not isinstance(key, str) or (not has_rates and not fixed_call and not upstream): + return Unpriced(usage_event_id=event.event_id, reason="pricing_inputs_missing") + info: Final = { + "key": key, + "litellm_provider": provider, + "mode": "chat", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "supported_openai_params": None, + **rates, + } + adjustments: Final = _fields(pricing.get("adjustments")) + discount: Final = adjustments.get("discount") + margin: Final = adjustments.get("margin") + secondary: Final = { + name: { + **info, + **_fields(_fields(snapshot).get("model_map_value")), + "key": _fields(snapshot).get("model_map_key") or name, + } + for name, snapshot in _fields(pricing.get("realtime_models")).items() + if _fields(snapshot).get("model_map_value") is not None + } + context: Final = RetainedPricing( + _json( + { + **secondary, + **{ + alias: info + for alias in dict.fromkeys((model, key, f"{provider}/{model}", model.removeprefix(f"{provider}/"))) + }, + } + ), + _json({provider: discount} if provider and discount is not None else {}), + _json({"global": margin} if margin is not None else {}), + ) + response_inputs: Final = _fields(pricing.get("response_inputs")) + tool_response: Final = _fields(pricing.get("tool_response")) + token_response: Final = ModelResponse( + model=model, + choices=[ + { + "message": { + "role": "assistant", + "annotations": tuple( + {"type": annotation} + for flag, annotation in (("web_search", "url_citation"),) + if tool_response.get(flag) + ), + } + } + ], + usage=Usage.model_validate(captured.usage or {}), + service_tier=response_inputs.get("service_tier"), + meta={"billed_units": response_inputs.get("billed_units")}, + hidden_params={ + "region_name": response_inputs.get("region_name"), + "audio_transcription_duration": response_inputs.get("audio_transcription_duration"), + }, + response_ms=logged.get("response_ms"), + ) + response: Final = ( + LiteLLMRealtimeStreamLoggingObject.model_validate( + { + "results": pricing.get("realtime_events", []), + "usage": captured.usage or {}, + "service_tier": response_inputs.get("service_tier"), + } + ) + if event.call_type in {CallTypes.arealtime, CallTypes.aresponses_websocket} + else ResponsesAPIResponse.model_validate( + { + "id": event.event_id, + "created_at": int(event.started_at.timestamp()), + "model": model, + "output": tool_response.get("output", []), + "usage": tool_response.get("usage"), + } + ) + if tool_response.get("responses_api") + else token_response + ) + logger: Final = Logging( + model=model, + messages=[], + stream=False, + call_type=event.call_type.value, + start_time=event.started_at, + litellm_call_id=event.event_id, + function_id=event.attempt_id, + ) + request: Final = _fields(pricing.get("request")) + deployment: Final = _fields(pricing.get("deployment")) + logger.update_environment_variables( # pyright: ignore[reportUnknownMemberType] # legacy Logging environment method accepts validated JSON fields + model=model, + custom_llm_provider=provider, + litellm_params={**deployment, **request, "metadata": {"model_info": _fields(pricing.get("model_info"))}}, + additional_response_cost=request.get("additional_response_cost"), + usage=Usage.model_validate(captured.usage or {}), + end_time=event.ended_at, + mcp_tool_call_metadata={ + "name": _fields(pricing.get("tool")).get("name"), + "mcp_server_cost_info": {"default_cost_per_query": _fields(pricing.get("tool")).get("cost_per_query")}, + }, + optional_params=dict(_fields(pricing.get("optional_params"))), + ) + if logged.get("tool_response_cost") is not None: + logger.model_call_details["response_cost"] = logged["tool_response_cost"] + logger.request_pricing = context if logged.get("custom_pricing") is True else None + logger.custom_cost_per_token = TypeAdapter(CostPerToken | None).validate_python(logged.get("custom_cost_per_token")) + logger.custom_cost_per_second = TypeAdapter(float | None).validate_python(logged.get("custom_cost_per_second")) + logger.standard_built_in_tools_params = TypeAdapter(StandardBuiltInToolsParams).validate_python( + { + "web_search_options": adjustments.get("web_search_options"), + "file_search": adjustments.get("file_search") or None, + } + ) + logger.record_provider_reported_cost( + TypeAdapter[float | None](float | None).validate_python(reported if upstream else None) + ) + billed_at: Final = TypeAdapter(datetime).validate_python(logged.get("current_billing_time", event.ended_at)) + if event.call_type is CallTypes.arealtime and captured.usage and captured.usage.get("metered_usage") is not None: + from litellm.constants import REALTIME_METERED_USAGE_KEY + from litellm.types.llms.xai import RealtimeMeteredUsage + + logger.model_call_details[REALTIME_METERED_USAGE_KEY] = TypeAdapter(RealtimeMeteredUsage).validate_python( + captured.usage["metered_usage"] + ) + logger.cost_billing_time = billed_at + with retained_pricing(context), pinned_billing_time(billed_at): + amount: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + if amount is None: + return Unpriced(usage_event_id=event.event_id, reason="calculation_failed") + retained_final: Final = captured.finalized_cost or {} + breakdown: Final = _fields(retained_final.get("cost_breakdown")) + guardrail: Final = TypeAdapter(float).validate_python(breakdown.get("guardrail_cost", 0.0)) + allowance: Final = TypeAdapter(float).validate_python(retained_final.get("allowance_discount", 0.0)) + if allowance and captured.allocation is None: + return Unpriced(usage_event_id=event.event_id, reason="allowance_allocation_missing") + finalized: Final = finalize_cost(amount + guardrail, logger.cost_breakdown, allowance) + if retained_final.get("response_cost") is not None and Decimal(str(finalized.amount)) != Decimal( + str(retained_final["response_cost"]) + ): + return Unpriced(usage_event_id=event.event_id, reason="finalized_cost_mismatch") + final_breakdown: Final = dict(finalized.breakdown or {}) + components: Final = { + name: Decimal(str(value)) + for name, value in final_breakdown.items() + if name in _COST_COMPONENTS and isinstance(value, (int, float)) + } + return PricingSnapshot.model_validate( + { + **components, + "calculator_version": "logging-v1", + "model": model, + "provider": provider, + "pricing_source": "upstream_header" + if upstream + else "per_request" + if request + else "deployment" + if deployment + else "cost_map", + "priced_model_key": key, + "total_cost": Decimal(str(finalized.amount)), + "guardrail_cost": Decimal(str(guardrail)), + "upstream_reported_cost": Decimal(str(logged["provider_reported_cost"])) if upstream else None, + "additional_costs": tuple( + {"name": name, "amount": Decimal(str(value))} + for name, value in { + **_fields(final_breakdown.get("additional_costs")), + **( + {"request": request["additional_response_cost"]} + if request.get("additional_response_cost") + else {} + ), + }.items() + ), + } + ) + + +_COST_COMPONENTS: Final = frozenset( + { + "input_cost", + "output_cost", + "cache_read_cost", + "cache_creation_cost", + "reasoning_cost", + "tool_usage_cost", + "original_cost", + "discount_percent", + "discount_amount", + "margin_percent", + "margin_fixed_amount", + "margin_total_amount", + } +) class LoggingCostCompiler: def __init__( self, ledger: AccountingLedger, - price_usage: Callable[[str], PricingSnapshot | Unpriced], - delivery_targets: tuple[DeliveryTarget, ...], + price_usage: Callable[[str], PricingSnapshot | Unpriced] = price_usage, + delivery_targets: tuple[DeliveryTarget, ...] | None = None, ) -> None: self._ledger: Final = ledger self._price_usage: Final = price_usage self._delivery_targets: Final = delivery_targets async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced: - return await self._ledger.compile_cost(event, self._price_usage, self._delivery_targets) + targets: Final = self._delivery_targets if self._delivery_targets is not None else cost_delivery_targets(event) + return await self._ledger.compile_cost(event, self._price_usage, targets) + + async def compile_pending(self, limit: int = 100) -> None: + for event in await self._ledger.pending_compilation(limit): + try: + await self.compile(event) + except Exception: # noqa: BLE001 # retain pending work for retry without starving other events + verbose_proxy_logger.exception("Cost compilation failed for %s", event.event_id) + + +_SPEND_TARGETS: Final = ( + (DimensionName.KEY, DeliveryTarget.KEY_SPEND), + (DimensionName.USER, DeliveryTarget.USER_SPEND), + (DimensionName.TEAM, DeliveryTarget.TEAM_SPEND), + (DimensionName.ORGANIZATION, DeliveryTarget.ORGANIZATION_SPEND), + (DimensionName.END_USER, DeliveryTarget.END_USER_SPEND), + (DimensionName.AGENT, DeliveryTarget.AGENT_SPEND), + (DimensionName.TAG, DeliveryTarget.TAG_SPEND), + (DimensionName.PROJECT, DeliveryTarget.PROJECT_SPEND), + (DimensionName.MODEL_ACCESS_GROUP, DeliveryTarget.MODEL_ACCESS_GROUP_SPEND), +) + + +def cost_delivery_targets(event: OutboundUsageEvent) -> tuple[DeliveryTarget, ...]: + names: Final = frozenset(d.name for d in event.dimensions) + return ( + *usage_delivery_targets(event.dimensions), + *(target for dimension, target in _SPEND_TARGETS if dimension in names), + *((DeliveryTarget.TEAM_MEMBER_SPEND,) if {DimensionName.TEAM, DimensionName.USER} <= names else ()), + *( + (DeliveryTarget.ORGANIZATION_MEMBER_SPEND,) + if {DimensionName.ORGANIZATION, DimensionName.USER} <= names + else () + ), + ) diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 19480a66678..c1d87d752d8 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.accounting import ( + CostCompiler, Dimension, DimensionName, InboundUsageEvent, @@ -240,6 +241,7 @@ _PRICING_FIELDS: Final = frozenset( "seconds", "search_context_size", "additional_response_cost", + "cost_per_query", } ) @@ -782,7 +784,13 @@ async def capture_proxy_response(logging_obj: object, response: object, *, backg timeout: Final = TypeAdapter(float).validate_python( settings.get("accounting_sync_ack_timeout_seconds", 20.0) ) - await asyncio.wait_for(asyncio.shield(captured.receipt), timeout=timeout) + async with asyncio.timeout(timeout): + await asyncio.shield(captured.receipt) + compiler: Final = usage_capture.compiler + if compiler is not None: + for write in captured.writes: + if isinstance(write.event, OutboundUsageEvent): + await compiler.compile(write.event) except Exception as error: # noqa: BLE001 # the configured response policy owns every capture failure verbose_proxy_logger.exception("Usage capture failed before the response boundary") if sync_ack and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed": @@ -866,6 +874,17 @@ class PricingCapture: def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[str, object]) -> PricingCapture: + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_pricing + + if logging_obj.request_pricing is not None: + with retained_pricing(logging_obj.request_pricing): + return _capture_effective_pricing(logging_obj, normalized, details) + return _capture_effective_pricing(logging_obj, normalized, details) + + +def _capture_effective_pricing( + logging_obj: Logging, normalized: object, details: Mapping[str, object] +) -> PricingCapture: import litellm from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -900,16 +919,38 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[ if builtins.get("web_search_options") is not None else None ), - "file_search": builtins.get("file_search") is not None, + "file_search": { + key: value + for key, value in _fields(builtins.get("file_search")).items() + if key in {"type", "storage_gb", "days"} + } + if builtins.get("file_search") is not None + else None, } ) - batch_usage: Final = _fields(_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model")) + realtime_models: Final = tuple( + dict.fromkeys( + chain.from_iterable(_realtime_pricing_models(event) for event in _realtime_cost_events(normalized)) + ) + ) + batch_usage: Final = _fields( + _response_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model") + ) return PricingCapture( _json(pricing), adjustments, _json( { **_pricing_inputs(details, logging_obj, normalized), + "realtime_models": { + model: StandardLoggingPayloadSetup.get_model_cost_information( + base_model=model, + custom_pricing=False, + custom_llm_provider=provider, + init_response_obj={"model": model}, + ) + for model in realtime_models + }, "batch_models": { model: StandardLoggingPayloadSetup.get_model_cost_information( base_model=model, @@ -926,10 +967,71 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[ ) +def _tool_cost_inputs(response: object) -> Mapping[str, object]: + from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import StandardBuiltInToolCostTracking + from litellm.types.llms.openai import ResponsesAPIResponse + + raw: Final = _response_fields(response) + flags: Final = { + "web_search": StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response), + "file_search": StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response), + } + if not isinstance(response, ResponsesAPIResponse) and raw.get("object") != "response": + return flags if any(flags.values()) else {} + return { + **flags, + "responses_api": True, + "usage": _response_fields(raw.get("usage")), + "output": tuple( + {key: value for key, value in _fields(item).items() if key in {"type", "id", "status", "usage"}} + for item in TypeAdapter(tuple[object, ...]).validate_python(raw.get("output") or ()) + ), + } + + +def _realtime_cost_fields(value: object) -> Mapping[str, JsonValue]: + fields: Final = _fields(value) + return { + **{ + key: _JSON_OBJECT.validate_python(fields[key]) for key in ("usage",) if isinstance(fields.get(key), Mapping) + }, + **{ + key: value + for key, value in fields.items() + if key in {"type", "id", "model", "service_tier"} and isinstance(value, str) + }, + **{ + key: dict(_realtime_cost_fields(fields[key])) + for key in ("response", "event", "session", "audio", "input", "transcription", "input_audio_transcription") + if isinstance(fields.get(key), Mapping) + }, + } + + +def _realtime_cost_events(response: object) -> tuple[Mapping[str, JsonValue], ...]: + results: Final = _response_fields(response).get("results") + return ( + tuple(_realtime_cost_fields(result) for result in TypeAdapter(tuple[object, ...]).validate_python(results)) + if isinstance(results, list) + else () + ) + + +def _realtime_pricing_models(value: Mapping[str, JsonValue]) -> Iterator[str]: + model: Final = value.get("model") + if isinstance(model, str): + yield model + for key in ("response", "event", "session", "audio", "input", "transcription", "input_audio_transcription"): + if isinstance(nested := value.get(key), dict): + yield from _realtime_pricing_models(nested) + + def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]: from litellm.litellm_core_utils.litellm_logging import Logging return { + "tool_response": _tool_cost_inputs(response), + "realtime_events": _realtime_cost_events(response), "request": _selected_pricing(kwargs), "deployment": _selected_pricing(_fields(kwargs.get("litellm_params"))), "optional_params": _selected_pricing(_fields(kwargs.get("optional_params"))), @@ -1006,6 +1108,7 @@ class UsageCaptureRuntime: self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle self._stopping = False self.meter: DurableUsageMeter | None = None + self.compiler: CostCompiler | None = None self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None def start( @@ -1014,6 +1117,7 @@ class UsageCaptureRuntime: settings: UsageCaptureSettings, *, load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None, + compiler: CostCompiler | None = None, ) -> None: from litellm.litellm_core_utils.litellm_logging import Logging @@ -1024,6 +1128,7 @@ class UsageCaptureRuntime: meter: Final = DurableUsageMeter(write_batch, settings) self.meter = meter self.load = load + self.compiler = compiler self._stopping = False Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage( meter, logger, response, start, end, freeze @@ -1082,3 +1187,4 @@ class UsageCaptureRuntime: Logging.usage_capture_callback = None self.meter = None self.load = None + self.compiler = None diff --git a/litellm/utils.py b/litellm/utils.py index b0826f84006..5fb27ad6ab6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7531,6 +7531,11 @@ def get_model_info( "supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"] } """ + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_model_info + + retained: Final = retained_model_info(model, custom_llm_provider) + if retained is not None: + return retained # api_key is a per-caller credential, not part of the model identity, so it is # kept out of the cache key; explicit keys are resolved without the cache. if api_key is not None: diff --git a/schema.prisma b/schema.prisma index 179b741fe7c..048e0e1e17a 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2111,9 +2111,12 @@ model LiteLLM_AccountingEvent { occurred_at DateTime recorded_at DateTime @default(now()) payload Json + compilation_state String @default("pending") + compilation_error String? @@index([recorded_at]) @@index([kind, source_event_id]) + @@index([kind, compilation_state, recorded_at]) } // One row per (event, target table, version): pending work for the projector and, diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index e107e8e130c..f783a128bc7 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -35,7 +35,7 @@ from litellm.accounting import ( ) from litellm.proxy.db import accounting_ledger from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite -from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler +from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler, price_usage from litellm.types.utils import CallTypes from tests.integration._support.database import scratch_database from tests.integration._support.prisma_clients import connected_proxy_prisma @@ -268,7 +268,7 @@ async def test_compilers_share_one_committed_charge_and_preserve_delivery_receip assert all(result == results[0] for result in results) assert isinstance(results[0], CostEvent) assert results[0].snapshot.total_cost == Decimal("0.125") - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(1,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)] deliveries: Final = _rows( ledger_database, 'SELECT target::text, state::text FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s', @@ -315,7 +315,7 @@ async def test_compilation_preserves_unknown_cost_without_creating_a_zero_charge assert await compiler.compile(event) == Unpriced(usage_event_id=event.event_id, reason="usage_not_committed") await _insert(store, event) assert await compiler.compile(event) == unresolved - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] @pytest.mark.asyncio @@ -329,7 +329,7 @@ async def test_compilation_rejects_an_unpriced_result_for_another_source(writer: ) with pytest.raises(AccountingConflictError, match="Unpriced result"): await compiler.compile(event) - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] @pytest.mark.asyncio @@ -341,7 +341,7 @@ async def test_compilation_rejects_a_cost_identity_collision(writer: _Writer, le await _insert(store, _usage(f"cost:{event.event_id}", inbound=True)) with pytest.raises(AccountingConflictError, match="identity_conflict"): await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] @pytest.mark.asyncio @@ -365,11 +365,11 @@ async def test_compilation_failure_rolls_back_and_can_be_retried(writer: _Writer await _insert(store, event) def failed_pricing(envelope: str) -> PricingSnapshot: - raise ValueError("malformed retained pricing") + raise RuntimeError("pricing temporarily unavailable") - with pytest.raises(ValueError, match="malformed retained pricing"): + with pytest.raises(RuntimeError, match="pricing temporarily unavailable"): await LoggingCostCompiler(store, failed_pricing, _TARGETS).compile(event) - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(0,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] result: Final = await LoggingCostCompiler(store, lambda _: _cost(amount="0").snapshot, _TARGETS).compile(event) assert isinstance(result, CostEvent) assert result.charges[0].value == 0 @@ -414,7 +414,7 @@ async def test_compiler_recovery_preserves_atomic_charge_and_delivery_acceptance with pytest.raises(error): await compiler.compile(event) count: Final = int(failure == "after_commit") - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(count,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(count,)] assert _rows( ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s', @@ -423,7 +423,7 @@ async def test_compiler_recovery_preserves_atomic_charge_and_delivery_acceptance result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) assert isinstance(result, CostEvent) assert result.snapshot == _cost().snapshot - assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent" WHERE kind = \'cost\'') == [(1,)] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)] @pytest.mark.asyncio @@ -786,3 +786,193 @@ async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit( assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)] assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] assert (meter.pending_bytes, meter.pending_records) == (0, 0) + + +@pytest.mark.asyncio +async def test_compilation_persists_invalid_pricing_reason_without_charging( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + accepted: Final = _rows( + ledger_database, + 'SELECT payload, payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = %s', + (event.event_id,), + ) + + def malformed(envelope: str) -> PricingSnapshot: + raise ValueError("invalid private input must not appear in durable diagnostics") + + result: Final = await LoggingCostCompiler(store, malformed, _TARGETS).compile(event) + assert result == Unpriced(usage_event_id=event.event_id, reason="invalid_pricing_inputs") + assert _rows( + ledger_database, + 'SELECT compilation_state, compilation_error FROM "LiteLLM_AccountingEvent" WHERE event_id = %s', + (event.event_id,), + ) == [("unpriced", "invalid_pricing_inputs")] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] + assert ( + _rows( + ledger_database, + 'SELECT payload, payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = %s', + (event.event_id,), + ) + == accepted + ) + async with connected_proxy_prisma(ledger_database) as restarted: + assert await AccountingLedger.for_client(restarted).pending_compilation() == () + + +@pytest.mark.asyncio +async def test_pending_compilation_only_returns_unresolved_billable_work(writer: _Writer, ledger_database: str) -> None: + _, store = writer + events: Final = tuple(_usage(name) for name in ("compiled", "excluded", "unknown", "pending")) + assert all(isinstance(event, OutboundUsageEvent) for event in events) + for event in events: + await _insert(store, event) + await _insert(store, _usage("inbound", inbound=True)) + compiled, excluded, unknown, pending = events + assert isinstance(compiled, OutboundUsageEvent) + assert isinstance(excluded, OutboundUsageEvent) + assert isinstance(unknown, OutboundUsageEvent) + assert isinstance(pending, OutboundUsageEvent) + assert isinstance( + await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(compiled), CostEvent + ) + await LoggingCostCompiler( + store, lambda _: Unpriced(usage_event_id=excluded.event_id, reason="not_charge_owner"), _TARGETS + ).compile(excluded) + await LoggingCostCompiler( + store, lambda _: Unpriced(usage_event_id=unknown.event_id, reason="usage_unknown"), _TARGETS + ).compile(unknown) + assert await store.pending_compilation(limit=1) == (pending,) + assert _rows( + ledger_database, + "SELECT event_id, compilation_state FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'outbound_usage' ORDER BY event_id", + ) == [("compiled", "compiled"), ("excluded", "excluded"), ("pending", "pending"), ("unknown", "unpriced")] + + +@pytest.mark.asyncio +async def test_restarted_compiler_recovers_transient_failures_without_revisiting_terminal_work( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + delayed: Final = _usage("delayed") + healthy: Final = _usage("healthy") + assert isinstance(delayed, OutboundUsageEvent) + assert isinstance(healthy, OutboundUsageEvent) + await _insert(store, delayed) + await _insert(store, healthy) + + def transient(envelope: str) -> PricingSnapshot: + if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == delayed.event_id: + raise RuntimeError("temporary pricing dependency failure") + return _cost().snapshot + + await LoggingCostCompiler(store, transient, _TARGETS).compile_pending() + assert await store.pending_compilation() == (delayed,) + assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ + (healthy.event_id,) + ] + async with connected_proxy_prisma(ledger_database) as restarted: + recovered: Final = AccountingLedger.for_client(restarted) + await LoggingCostCompiler(recovered, lambda _: _cost().snapshot, _TARGETS).compile_pending() + assert await recovered.pending_compilation() == () + await LoggingCostCompiler(recovered, transient, _TARGETS).compile_pending() + assert _rows( + ledger_database, + "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost' ORDER BY source_event_id", + ) == [(delayed.event_id,), (healthy.event_id,)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_ack", (False, True)) +async def test_proxy_capture_and_background_compiler_converge_on_final_header_charge( + writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, sync_ack: bool +) -> None: + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.spend_tracking.cost_finalization import finalize_response_cost + from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime, capture_proxy_response + from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + + _, store = writer + entered: Final = asyncio.Event() + release: Final = asyncio.Event() + batches: Final = UsageBatchWriter(store.insert_usage_batch, store.load_usage_batch) + + async def delayed_write(writes: tuple[UsageWrite, ...]) -> None: + entered.set() + await release.wait() + await batches.write(writes) + + compiler: Final = LoggingCostCompiler(store) + runtime: Final = UsageCaptureRuntime() + runtime.start(delayed_write, UsageCaptureSettings(batch_size=2), load=store.load_usage_batch, compiler=compiler) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": sync_ack}) + logger: Final = Logging( + model="capture-priced-model", + messages=[], + stream=False, + call_type="acompletion", + start_time=_AT, + litellm_call_id="gateway-request", + function_id="gateway-attempt", + ) + logger.update_environment_variables( + model="capture-priced-model", custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + logger.set_request_pricing( + "capture-priced-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} + ) + response: Final = ModelResponse( + id="gateway-provider", model="capture-priced-model", usage=Usage(prompt_tokens=10, completion_tokens=4) + ) + try: + finalized: Final = await finalize_response_cost(logger, response, None, _AT, guardrail_cost=0.03) + assert finalized is not None + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), response_cost=finalized.amount, litellm_logging_obj=logger + ) + capture: Final = asyncio.create_task(capture_proxy_response(logger, response)) + await asyncio.wait_for(entered.wait(), timeout=5) + if sync_ack: + assert not capture.done() + else: + await asyncio.wait_for(asyncio.shield(capture), timeout=5) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)] + release.set() + await asyncio.wait_for(capture, timeout=10) + assert runtime.meter is not None + await runtime.meter.flush() + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ + (1 if sync_ack else 0,) + ] + await asyncio.gather(*(compiler.compile_pending() for _ in range(3))) + costs: Final = _rows( + ledger_database, "SELECT payload::text FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'" + ) + assert len(costs) == 1 + cost_envelope: Final = json.loads(str(costs[0][0])) + committed: Final = CostEvent.model_validate_json(json.dumps(cost_envelope["event"])) + assert committed.snapshot.total_cost == Decimal(headers["x-litellm-response-cost"]) + assert float(committed.snapshot.total_cost) == pytest.approx(0.21) + monkeypatch.setattr(litellm, "model_cost", {"openai/capture-priced-model": {"input_cost_per_token": 99.0}}) + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.99}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 999.0}}) + replayed: Final = price_usage(json.dumps(cost_envelope["pricing_inputs"])) + assert replayed == committed.snapshot + assert litellm.model_cost["openai/capture-priced-model"]["input_cost_per_token"] == 99.0 + await capture_proxy_response(logger, response) + await compiler.compile_pending() + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)] + finally: + release.set() + await runtime.stop(5) diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py new file mode 100644 index 00000000000..bd093d6e012 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -0,0 +1,849 @@ +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Final + +import pytest + +from litellm.accounting import Dimension, DimensionName, OutboundUsageEvent, Outcome, PricingSnapshot, Unpriced +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.proxy.spend_tracking.cost_compiler import price_usage +from litellm.types.utils import CallTypes + +_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _envelope() -> dict[str, object]: + event: Final = OutboundUsageEvent( + event_id="attempt", + attempt_id="attempt", + request_id="request", + retry_of_attempt_id=None, + call_type=CallTypes.acompletion, + started_at=_AT, + ended_at=_AT, + outcome=Outcome.SUCCEEDED, + dimensions=( + Dimension(name=DimensionName.MODEL, value="openai/retained-model"), + Dimension(name=DimensionName.PROVIDER, value="openai"), + ), + metrics=None, + ) + return { + "event": event.model_dump(mode="json"), + "charge_eligible": True, + "usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14}, + "pricing": { + "model_map_information": { + "model_map_key": "openai/retained-model", + "model_map_value": { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "litellm_provider": "openai", + "mode": "chat", + }, + }, + "adjustments": {"discount": 0.1, "margin": {"fixed_amount": 0.008}}, + }, + } + + +def test_compiler_prices_retained_rates_without_registering_a_model() -> None: + import litellm + + result: Final = price_usage(json.dumps(_envelope())) + assert isinstance(result, PricingSnapshot) + assert result.total_cost == Decimal(str((10 * 0.01 + 4 * 0.02) * 0.9 + 0.008)) + assert "openai/retained-model" not in litellm.model_cost + + +@pytest.mark.parametrize( + ("field", "value", "reason"), + [ + ("usage", None, "usage_unknown"), + ("charge_eligible", False, "not_charge_owner"), + ("pricing", {}, "pricing_inputs_missing"), + ("finalized_cost", {"response_cost": 999}, "finalized_cost_mismatch"), + ], +) +def test_compiler_does_not_invent_a_charge(field: str, value: object, reason: str) -> None: + result: Final = price_usage(json.dumps({**_envelope(), field: value})) + assert result == Unpriced(usage_event_id="attempt", reason=reason) + + +def _retained_envelope( + *, + rates: dict[str, object] | None = None, + usage: dict[str, object] | None = None, + logging: dict[str, object] | None = None, + response_inputs: dict[str, object] | None = None, + deployment: dict[str, object] | None = None, + request: dict[str, object] | None = None, +) -> dict[str, object]: + from pydantic import TypeAdapter + + original: Final = _envelope() + pricing: Final = TypeAdapter(dict[str, object]).validate_python(original["pricing"]) + mapped: Final = TypeAdapter(dict[str, object]).validate_python(pricing["model_map_information"]) + base_rates: Final = TypeAdapter(dict[str, object]).validate_python(mapped["model_map_value"]) + return { + **original, + "usage": usage if usage is not None else original["usage"], + "pricing": { + **pricing, + "model_map_information": {**mapped, "model_map_value": {**base_rates, **(rates or {})}}, + "logging": logging or {}, + "response_inputs": response_inputs or {}, + "deployment": deployment or {}, + "request": request or {}, + "adjustments": {"discount": None, "margin": None}, + }, + } + + +def test_retained_pricing_ignores_later_global_price_and_adjustment_changes(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + + captured: Final = json.dumps(_envelope()) + original: Final = price_usage(captured) + monkeypatch.setattr(litellm, "model_cost", {"openai/retained-model": {"input_cost_per_token": 99.0}}) + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.99}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 999.0}}) + recomputed: Final = price_usage(captured) + assert isinstance(original, PricingSnapshot) + assert recomputed == original + assert litellm.model_cost["openai/retained-model"]["input_cost_per_token"] == 99.0 + + +def test_repricing_recomputes_retained_usage_instead_of_returning_a_reported_total() -> None: + from pydantic import TypeAdapter + + captured: Final = _retained_envelope() + pricing: Final = TypeAdapter(dict[str, object]).validate_python(captured["pricing"]) + original: Final = price_usage(json.dumps({**captured, "pricing": {**pricing, "reported_cost": 999.0}})) + changed: Final = price_usage(json.dumps({**captured, "usage": {"prompt_tokens": 20, "completion_tokens": 8}})) + assert isinstance(original, PricingSnapshot) + assert isinstance(changed, PricingSnapshot) + assert float(original.total_cost) == pytest.approx(0.18) + assert float(changed.total_cost) == pytest.approx(float(original.total_cost) * 2) + + +def test_retained_token_pricing_preserves_cache_reasoning_and_service_tier() -> None: + captured: Final = _retained_envelope( + rates={ + "input_cost_per_token_priority": 0.03, + "output_cost_per_token_priority": 0.05, + "cache_read_input_token_cost_priority": 0.002, + "output_cost_per_reasoning_token_priority": 0.07, + }, + usage={ + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": {"cached_tokens": 40}, + "completion_tokens_details": {"reasoning_tokens": 5}, + }, + response_inputs={"service_tier": "priority"}, + ) + result: Final = price_usage(json.dumps(captured)) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(60 * 0.03 + 40 * 0.002 + 15 * 0.05 + 5 * 0.07) + + +def test_upstream_reported_price_takes_precedence_without_custom_pricing() -> None: + result: Final = price_usage(json.dumps(_retained_envelope(logging={"provider_reported_cost": 0.037}))) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.037) + + +def test_per_request_pricing_overrides_upstream_reported_price() -> None: + result: Final = price_usage( + json.dumps( + _retained_envelope( + logging={ + "provider_reported_cost": 0.037, + "custom_pricing": True, + "custom_cost_per_token": {"input_cost_per_token": 0.04, "output_cost_per_token": 0.06}, + } + ) + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.04 + 4 * 0.06) + + +def test_retained_regional_pricing_preserves_data_residency_uplift() -> None: + result: Final = price_usage( + json.dumps( + _retained_envelope( + rates={"regional_processing_uplift_multiplier_eu": 1.1}, + deployment={"data_residency": "eu"}, + ) + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx((10 * 0.01 + 4 * 0.02) * 1.1) + + +def test_retained_request_additional_cost_is_applied_once() -> None: + result: Final = price_usage(json.dumps(_retained_envelope(request={"additional_response_cost": 0.3}))) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.01 + 4 * 0.02 + 0.3) + + +def _capture_logger(call_type: str = "acompletion", *, stream: bool = False) -> Logging: + from tests.unit.proxy.spend_tracking.test_usage_capture import _logger + + logger: Final = _logger(call_type, stream=stream) + logger.set_request_pricing("capture-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}) + return logger + + +@pytest.mark.asyncio +async def test_real_capture_preserves_synchronous_finalization_and_guardrail_parity() -> None: + from litellm.proxy.spend_tracking.cost_finalization import finalize_response_cost + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger() + response: Final = ModelResponse( + id="finalized", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14) + ) + finalized: Final = await finalize_response_cost(logger, response, None, _AT, guardrail_cost=0.03) + assert finalized is not None + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(finalized.amount) + assert float(result.guardrail_cost or 0) == pytest.approx(0.03) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_stream_capture_only_prices_final_usage() -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, ModelResponseStream, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(stream=True) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, ModelResponseStream(id="chunk"), _AT, _AT) + assert meter.pending_records == 0 + capture_logging_usage( + meter, + logger, + ModelResponse(id="terminal", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)), + _AT, + _AT, + ) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.18) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_internal_capture_excludes_usage_owned_by_parent() -> None: + from litellm._internal_context import internal_sub_call + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + with internal_sub_call("parent"): + capture_logging_usage( + meter, + _capture_logger(), + ModelResponse(id="child", model="capture-model", usage=Usage(prompt_tokens=10)), + _AT, + _AT, + ) + await meter.close() + assert price_usage(database.rows[-1].envelope) == Unpriced( + usage_event_id=database.rows[-1].event.event_id, reason="not_charge_owner" + ) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("already_calculated", (False, True)) +async def test_real_mcp_capture_prices_retained_tool_fee_without_token_usage(already_calculated: bool) -> None: + from mcp.types import CallToolResult, TextContent + + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(CallTypes.call_mcp_tool.value) + logger.model_call_details["mcp_tool_call_metadata"] = { + "name": "lookup", + "namespaced_tool_name": "server.lookup", + "mcp_server_cost_info": {"default_cost_per_query": 0.2, "tool_name_to_cost_per_query": {"lookup": 0.3}}, + } + if already_calculated: + logger.model_call_details["response_cost"] = 0.3 + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, CallToolResult(content=[TextContent(type="text", text="ok")]), _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.3) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_agent_capture_prices_retained_per_query_fee() -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.agents import LiteLLMSendMessageResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(CallTypes.asend_message.value) + logger.update_environment_variables( + model="capture-model", custom_llm_provider="a2a", litellm_params={"cost_per_query": 0.4}, optional_params={} + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, LiteLLMSendMessageResponse(id="agent"), _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.4) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reported_cost", (0.0, 0.123)) +async def test_real_capture_prices_upstream_cost_without_a_model_map(reported_cost: float) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger() + logger.record_provider_reported_cost(reported_cost) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage( + meter, logger, ModelResponse(id="upstream", model="capture-model", usage=Usage(prompt_tokens=10)), _AT, _AT + ) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(reported_cost) + assert result.upstream_reported_cost == Decimal(str(reported_cost)) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancelled", (False, True)) +async def test_real_failure_capture_preserves_incurred_charge(cancelled: bool) -> None: + from fastapi import HTTPException + + from litellm.litellm_core_utils.litellm_logging import UsageObservation + from litellm.proxy.spend_tracking.usage_capture import capture_callback, capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger() + usage: Final = Usage(prompt_tokens=4, completion_tokens=0, total_tokens=4) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage( + meter, logger, ModelResponse(model="capture-model", usage=usage), _AT, _AT, freeze_only=True + ) + capture_callback( + meter, + { + **logger.model_call_details, + "litellm_logging_obj": logger, + "combined_usage_object": usage, + "_accounting_usage_observed": UsageObservation.KNOWN, + "log_event_type": "failed_api_call", + }, + HTTPException(499, "Client disconnected") if cancelled else RuntimeError("provider failed after billing"), + _AT, + _AT, + ) + await meter.close() + outbound: Final = database.rows[-1] + assert outbound.event.outcome is (Outcome.CANCELLED if cancelled else Outcome.FAILED) + result: Final = price_usage(outbound.envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.04) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_captured_price_replay_uses_original_off_peak_clock(monkeypatch: pytest.MonkeyPatch) -> None: + from datetime import timedelta + + import litellm + from litellm._internal_context import pinned_billing_time + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger() + monkeypatch.setitem( + litellm.model_cost, + "capture-model", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "off_peak_pricing": { + "hours_utc": "00:00-01:00", + "input_cost_per_token": 0.005, + "output_cost_per_token": 0.01, + }, + }, + ) + response: Final = ModelResponse(model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + with pinned_billing_time(_AT): + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + with pinned_billing_time(_AT + timedelta(hours=3)): + replay: Final = price_usage(database.rows[-1].envelope) + assert source_cost == pytest.approx(0.09) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_realtime_capture_preserves_audio_and_text_token_prices() -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(CallTypes.arealtime.value) + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_audio_token": 0.03, + "output_cost_per_audio_token": 0.04, + }, + ) + events: Final = [ + { + "type": "response.done", + "response": { + "id": "turn", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_token_details": {"text_tokens": 4, "audio_tokens": 6, "cached_tokens": 0}, + "output_token_details": {"text_tokens": 2, "audio_tokens": 3}, + }, + "output": "private transcript excluded from accounting", + }, + } + ] + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, events, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private transcript" not in captured + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(4 * 0.01 + 6 * 0.03 + 2 * 0.02 + 3 * 0.04) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_realtime_capture_preserves_metered_audio_and_message_fees() -> None: + from litellm.constants import REALTIME_METERED_USAGE_KEY + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.xai import RealtimeMeteredUsage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(CallTypes.arealtime.value) + logger.update_environment_variables( + model="capture-model", custom_llm_provider="xai", litellm_params={}, optional_params={} + ) + logger.set_request_pricing( + "capture-model", + "xai", + { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_second": 0.02, + "input_cost_per_text_message": 0.1, + }, + ) + logger.model_call_details[REALTIME_METERED_USAGE_KEY] = RealtimeMeteredUsage(True, 1.5, 2.5, 3) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage( + meter, + logger, + [ + { + "type": "response.done", + "response": { + "id": "metered", + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + }, + } + ], + _AT, + _AT, + ) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx((1.5 + 2.5) * 0.02 + 3 * 0.1) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_batch_rows_keep_per_request_tiers_across_repeated_terminal_retrieval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.batches.batch_utils import batch_cost_and_usage + from litellm.proxy.spend_tracking.usage_capture import ( + UsageBatchWriter, + capture_batch_callback, + capture_logging_usage, + ) + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + rates: Final = { + "input_cost_per_token_batches": 0.001, + "input_cost_per_token_above_200k_tokens_batches": 0.002, + "output_cost_per_token_batches": 0.0, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + rows: Final = [ + { + "response": { + "status_code": 200, + "body": { + "model": "capture-model", + "usage": {"prompt_tokens": tokens, "completion_tokens": 0, "total_tokens": tokens}, + }, + } + } + for tokens in (10, 200001) + ] + billed: Final = batch_cost_and_usage(rows, "openai") + response: Final = {"id": "batch", "status": "completed", "usage": billed.usage} + database: Final = StoredUsage() + writer: Final = UsageBatchWriter(database.insert, database.load) + meter: Final = DurableUsageMeter(writer.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) + logger: Final = _logger("aretrieve_batch") + logger.model_call_details["response_cost"] = billed.cost + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + await meter.flush() + first: Final = tuple(database.rows) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + await meter.close() + assert tuple(database.rows) == first + assert price_usage(database.rows[0].envelope) == Unpriced( + usage_event_id=database.rows[0].event.event_id, reason="not_charge_owner" + ) + costs: Final = tuple(price_usage(row.envelope) for row in database.rows[1:]) + assert all(isinstance(cost, PricingSnapshot) for cost in costs), costs + assert sum(float(cost.total_cost) for cost in costs if isinstance(cost, PricingSnapshot)) == pytest.approx( + 10 * 0.001 + 200001 * 0.002 + ) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transcription", (False, True)) +async def test_realtime_capture_replays_sanitized_lifecycle_and_transcription_charges(transcription: bool) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(CallTypes.arealtime.value) + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_second": 0.02, + }, + ) + events: Final = ( + [ + { + "type": "transcription_session.created", + "session": {"input_audio_transcription": {"model": "capture-model"}}, + }, + { + "type": "conversation.item.input_audio_transcription.completed", + "item_id": "audio", + "transcript": "private transcript", + "usage": {"type": "duration", "seconds": 2.5}, + }, + ] + if transcription + else [ + {"type": "session.started", "session": {"model": "capture-model"}}, + {"type": "session.closed", "usage": {"seconds": 2.5}, "transcript": "private transcript"}, + ] + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, events, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private transcript" not in captured + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(2.5 * 0.02) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("file_search", (False, True)) +async def test_capture_replays_builtin_search_charge_without_retaining_results(file_search: bool) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger("aresponses") + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "search_context_cost_per_query": {"search_context_size_medium": 0.1}, + "file_search_cost_per_1k_calls": 0.025, + }, + ) + logger.standard_built_in_tools_params = ( + {"file_search": {"type": "file_search", "vector_store_ids": ["store"]}} + if file_search + else {"web_search_options": {"search_context_size": "medium"}} + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "search", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "capture-model", + "output": [ + {"type": "file_search_call", "id": "search", "queries": ["private query"], "status": "completed"} + ] + if file_search + else [ + { + "type": "web_search_call", + "id": "search", + "status": "completed", + "action": {"type": "search", "query": "private query"}, + } + ], + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + } + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost is not None and source_cost > 0.18 + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private query" not in captured + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_capture_replays_multi_round_aggregate_with_one_fixed_request_margin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.responses.mcp.round_billing import billed_for_every_round, summed_cost_breakdown + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.1}}) + first_logger: Final = _capture_logger() + final_logger: Final = _capture_logger() + first: Final = ModelResponse(id="first", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + final: Final = ModelResponse(id="final", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + first._hidden_params["response_cost"] = first_logger.response_cost_calculator(first, ignore_stamped_cost=True) + final._hidden_params["response_cost"] = final_logger.response_cost_calculator(final, ignore_stamped_cost=True) + aggregate: Final = billed_for_every_round(first, first_logger.cost_breakdown, final, final_logger.cost_breakdown) + breakdown: Final = summed_cost_breakdown(first, first_logger.cost_breakdown, final, final_logger.cost_breakdown) + final_logger.cost_breakdown = breakdown + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, final_logger, aggregate, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(2 * (10 * 0.01 + 4 * 0.02) + 0.1) + assert float(result.total_cost) == pytest.approx(aggregate._hidden_params["response_cost"]) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("reported", (-0.1, float("inf"), float("-inf"), float("nan"), True, False)) +@pytest.mark.parametrize("has_rates", (False, True)) +def test_invalid_upstream_price_cannot_become_an_authoritative_charge(reported: float, has_rates: bool) -> None: + from pydantic import TypeAdapter + + envelope: Final = _retained_envelope(logging={"provider_reported_cost": reported}) + pricing: Final = TypeAdapter(dict[str, object]).validate_python(envelope["pricing"]) + if not has_rates: + pricing["model_map_information"] = {} + envelope["pricing"] = pricing + result: Final = price_usage(json.dumps(envelope)) + if has_rates: + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.18) + assert result.upstream_reported_cost is None + else: + assert result == Unpriced(usage_event_id="attempt", reason="pricing_inputs_missing") + + +def test_repricing_rejects_unretained_allowance_allocation() -> None: + envelope: Final = _retained_envelope() + envelope["finalized_cost"] = {"allowance_discount": 0.1} + assert price_usage(json.dumps(envelope)) == Unpriced( + usage_event_id="attempt", reason="allowance_allocation_missing" + ) + + +@pytest.mark.asyncio +async def test_retained_pricing_is_task_local_and_resets_on_cancellation() -> None: + import asyncio + + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import ( + RetainedPricing, + pricing_discounts, + retained_model_info, + retained_models, + retained_pricing, + ) + + outer: Final = RetainedPricing("{}", '{"openai":0.1}', "{}") + inner: Final = RetainedPricing("{}", '{"openai":0.9}', "{}") + ready: Final = asyncio.Event() + hold: Final = asyncio.Event() + restored: Final = asyncio.Event() + + async def child() -> None: + try: + with retained_pricing(inner): + assert pricing_discounts()["openai"] == 0.9 + with pytest.raises(ValueError, match="no retained model"): + retained_model_info("gpt-4o-mini", "openai") + ready.set() + await hold.wait() + finally: + assert pricing_discounts()["openai"] == 0.1 + restored.set() + + assert retained_models() is None + with retained_pricing(outer): + task: Final = asyncio.create_task(child()) + try: + await asyncio.wait_for(ready.wait(), timeout=5) + assert pricing_discounts()["openai"] == 0.1 + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert restored.is_set() + assert pricing_discounts()["openai"] == 0.1 + assert retained_models() is None + assert retained_model_info("gpt-4o-mini", "openai") is None + + +def test_incomplete_retained_rates_remain_unpriced() -> None: + result: Final = price_usage( + json.dumps(_retained_envelope(rates={"input_cost_per_token": None, "output_cost_per_token": None})) + ) + assert result == Unpriced(usage_event_id="attempt", reason="pricing_inputs_missing") + + +def test_explicit_zero_retained_token_rates_are_known_free() -> None: + result: Final = price_usage( + json.dumps(_retained_envelope(rates={"input_cost_per_token": 0.0, "output_cost_per_token": 0.0})) + ) + assert isinstance(result, PricingSnapshot) + assert result.total_cost == Decimal("0") + assert result.pricing_source == "cost_map" diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index 32710e594ee..24bb3480560 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1,13 +1,13 @@ import asyncio import json -from collections.abc import Awaitable, Mapping +from collections.abc import Mapping from datetime import datetime, timezone from decimal import Decimal from typing import Final import pytest -from litellm.accounting import Metric, MetricName, Outcome +from litellm.accounting import Metric, MetricName, OutboundUsageEvent, Outcome, Unpriced from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings @@ -441,13 +441,13 @@ async def test_supported_sync_ack_waits_for_commit(call_type: str, monkeypatch: monkeypatch.setattr(proxy_server, "usage_capture", runtime) monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) deadlines: Final = asyncio.Queue[float | None]() - original_wait: Final = asyncio.wait_for + original_timeout: Final = asyncio.timeout - async def wait_with_deadline(awaitable: Awaitable[object], timeout: float | None) -> object: - deadlines.put_nowait(timeout) - return await original_wait(awaitable, timeout) + def timeout_with_deadline(delay: float | None) -> asyncio.Timeout: + deadlines.put_nowait(delay) + return original_timeout(delay) - monkeypatch.setattr(asyncio, "wait_for", wait_with_deadline) + monkeypatch.setattr(asyncio, "timeout", timeout_with_deadline) response: Final = asyncio.create_task( capture_proxy_response( _logger(call_type), @@ -1075,17 +1075,14 @@ async def test_response_ack_timeout_preserves_success_and_later_commits(monkeypa runtime.start(database.write, UsageCaptureSettings(batch_size=2)) monkeypatch.setattr(proxy_server, "usage_capture", runtime) monkeypatch.setattr(proxy_server, "prisma_client", None) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) + monkeypatch.setattr( + proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.01} + ) logger: Final = _logger() - async def expired_deadline(future: object, timeout: float) -> None: - raise TimeoutError("controlled response deadline") - try: - with monkeypatch.context() as clock: - clock.setattr(asyncio, "wait_for", expired_deadline) - with pytest.raises(HTTPException) as error: - await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3))) + with pytest.raises(HTTPException) as error: + await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3))) assert error.value.status_code == 503 await database.entered.wait() captured: Final = logger.model_call_details["_accounting_capture"] @@ -1710,3 +1707,50 @@ async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> Non assert runtime.meter is not None and runtime.meter.pending_bytes == 0 finally: await runtime.stop(1) + + +class DelayedCostCompiler: + def __init__(self) -> None: + self.entered: Final = asyncio.Event() + self.cancelled: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def compile(self, event: OutboundUsageEvent) -> Unpriced: + self.entered.set() + try: + await self.release.wait() + except asyncio.CancelledError: + self.cancelled.set() + raise + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") + + +@pytest.mark.asyncio +async def test_sync_ack_deadline_bounds_compilation_after_usage_commit(monkeypatch: pytest.MonkeyPatch) -> None: + from fastapi import HTTPException + + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response + from litellm.types.utils import ModelResponse, Usage + + database: Final = CommitBarrier() + database.release.set() + compiler: Final = DelayedCostCompiler() + runtime: Final = UsageCaptureRuntime() + runtime.start(database.write, UsageCaptureSettings(batch_size=2), compiler=compiler) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + monkeypatch.setattr( + proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.1} + ) + try: + with pytest.raises(HTTPException) as error: + await asyncio.wait_for( + capture_proxy_response(_logger(), ModelResponse(id="provider", usage=Usage(prompt_tokens=3))), 2 + ) + assert error.value.status_code == 503 + assert compiler.entered.is_set() + assert compiler.cancelled.is_set() + assert database.committed == ("caller:inbound", "provider") + finally: + compiler.release.set() + await runtime.stop(1) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index cca138e0bb5..bb4035e8725 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -11108,3 +11108,50 @@ def test_upstream_cost_header_total_on_an_unmapped_alias_is_split_by_the_served_ input_share: Final = (10 * 0.001) / (10 * 0.001 + 20 * 0.002) assert breakdown["input_cost"] == pytest.approx(0.25 * input_share) assert breakdown["output_cost"] == pytest.approx(0.25 * (1 - input_share)) + + +@pytest.mark.parametrize( + ("model", "retained_key", "deployment_rate"), + ( + ("video-model", "openai/video-model", None), + ("openai/video-model", "video-model", None), + ("openai/video-model", "openai/video-model", 0.7), + ), +) +def test_video_pricing_uses_retained_catalog_and_deployment_precedence( + monkeypatch: pytest.MonkeyPatch, model: str, retained_key: str, deployment_rate: float | None +) -> None: + from pydantic import TypeAdapter + + from litellm.cost_calculator import default_video_cost_calculator + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing + from litellm.types.utils import ModelInfo + + info: Final = TypeAdapter(ModelInfo).validate_python( + { + "key": retained_key, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": "openai", + "mode": "image_generation", + "supported_openai_params": None, + "output_cost_per_video_per_second": 0.2, + } + ) + retained: Final = RetainedPricing(json.dumps({retained_key: info}), "{}", "{}") + changed: Final = {**info, "output_cost_per_video_per_second": 9.0} + monkeypatch.setattr(litellm, "model_cost", {"video-model": changed, "openai/video-model": changed}) + deployment: Final = ( + TypeAdapter(ModelInfo).validate_python({**info, "output_cost_per_video_per_second": deployment_rate}) + if deployment_rate is not None + else None + ) + with retained_pricing(retained): + price: Final = default_video_cost_calculator( + model, duration_seconds=4, custom_llm_provider="openai", model_info=deployment + ) + assert price == pytest.approx(4 * (deployment_rate if deployment_rate is not None else 0.2)) + assert litellm.model_cost["video-model"]["output_cost_per_video_per_second"] == 9.0 diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 35fbd28eae7..9ee21032343 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1,13 +1,4 @@ -""" -Test that register_model() in completion() and embedding() passes all -custom pricing fields from kwargs and model_info, not just the base -input/output costs. - -Previously, only input_cost_per_token, output_cost_per_token, and -litellm_provider were forwarded. Fields like cache_read_input_token_cost, -mode, and supports_prompt_caching were dropped, causing incorrect cost -calculations for DB-sourced models with prompt caching pricing. -""" +"""Verify explicit model registration and isolated request pricing preserve effective rates.""" import copy import itertools @@ -17,9 +8,9 @@ from typing import Final, NoReturn import pytest - import litellm from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.main import _build_custom_pricing_entry from litellm.utils import _invalidate_model_cost_lowercase_map @@ -236,10 +227,7 @@ def test_register_model_strips_none_litellm_provider(): registered = litellm.model_cost.get(model_key) assert registered is not None, f"{model_key} should be in model_cost" # The key may be absent entirely, but if present it must not be None. - assert ( - "litellm_provider" not in registered - or registered["litellm_provider"] is not None - ) + assert "litellm_provider" not in registered or registered["litellm_provider"] is not None # Downstream consumers must accept this entry for any provider, # mirroring what the cost calculator does. assert _check_provider_match(registered, "openai") is True @@ -297,9 +285,7 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp # The strip must have removed the None-valued provider that # ``get_model_info`` returned. The key may be absent entirely, but # it must never be present with value ``None``. - assert "litellm_provider" not in registered or ( - registered["litellm_provider"] is not None - ), ( + assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), ( "register_model failed to strip litellm_provider=None returned " f"by get_model_info, got {registered.get('litellm_provider')!r}" ) @@ -360,14 +346,8 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa ) registered = litellm.model_cost[registered_key] - assert ( - registered.get("cache_creation_input_token_cost") - == builtin["cache_creation_input_token_cost"] - ) - assert ( - registered.get("cache_read_input_token_cost") - == builtin["cache_read_input_token_cost"] - ) + assert registered.get("cache_creation_input_token_cost") == builtin["cache_creation_input_token_cost"] + assert registered.get("cache_read_input_token_cost") == builtin["cache_read_input_token_cost"] assert registered["litellm_provider"] == "bedrock" usage = Usage( @@ -431,8 +411,7 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog): ) assert any( - registered_key in record.message - and "cache_creation_input_token_cost" in record.message + registered_key in record.message and "cache_creation_input_token_cost" in record.message for record in caplog.records ), "expected a warning naming the unmapped key and the cache cost fields" finally: @@ -619,17 +598,14 @@ def test_register_model_router_add_deployment_custom_pricing_applies(): # the deployment's provider. from litellm.utils import _check_provider_match - registered_keys = [ - k for k in (deployment_model, model_key) if k in litellm.model_cost - ] + registered_keys = [k for k in (deployment_model, model_key) if k in litellm.model_cost] assert registered_keys, ( - "Router.add_deployment did not register custom pricing for " - f"{model_key} / {deployment_model}" + f"Router.add_deployment did not register custom pricing for {model_key} / {deployment_model}" ) for k in registered_keys: - assert ( - _check_provider_match(litellm.model_cost[k], "openai") is True - ), f"custom pricing for {k} was dropped by _check_provider_match" + assert _check_provider_match(litellm.model_cost[k], "openai") is True, ( + f"custom pricing for {k} was dropped by _check_provider_match" + ) finally: litellm.model_cost.pop(model_key, None) litellm.model_cost.pop(deployment_model, None) @@ -646,17 +622,15 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): """ shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 + logger: Final = _request_logger(shared_key, "embedding") try: - litellm.embedding( + response = litellm.embedding( model=shared_key, + litellm_logging_obj=logger, input=["hello"], api_key="fake-key", input_cost_per_token=0.0, @@ -666,12 +640,10 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response=[0.1, 0.2], ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 - assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == 0.0 sibling_response = litellm.embedding( model=shared_key, @@ -679,9 +651,7 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): api_key="fake-key", mock_response=[0.1, 0.2], ) - sibling_cost = litellm.completion_cost( - completion_response=sibling_response, call_type="embedding" - ) + sibling_cost = litellm.completion_cost(completion_response=sibling_response, call_type="embedding") assert sibling_cost == pytest.approx(10 * builtin_input_cost) finally: _restore_model_cost_entries(snapshot) @@ -695,17 +665,15 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-custom" override_input_cost = 5e-05 - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost != override_input_cost + logger: Final = _request_logger(shared_key, "embedding") try: response = litellm.embedding( model=shared_key, + litellm_logging_obj=logger, input=["hello"], api_key="fake-key", input_cost_per_token=override_input_cost, @@ -715,19 +683,9 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): mock_response=[0.1, 0.2], ) - request_cost = litellm.completion_cost( - completion_response=response, - model=shared_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - router_model_id=deployment_id, - ) + request_cost = logger.response_cost_calculator(response, ignore_stamped_cost=True) assert request_cost == pytest.approx(10 * override_input_cost) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ) + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost finally: _restore_model_cost_entries(snapshot) @@ -736,17 +694,15 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): """Same isolation as the embedding path, exercised through completion().""" shared_key = "openai/gpt-4o-mini" deployment_id = "lit3991-wildcard-chat-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "gpt-4o-mini", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "gpt-4o-mini", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 + logger: Final = _request_logger(shared_key, "completion") try: - litellm.completion( + response = litellm.completion( model=shared_key, + litellm_logging_obj=logger, messages=[{"role": "user", "content": "hello"}], api_key="fake-key", input_cost_per_token=0.0, @@ -756,47 +712,43 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response="hello back", ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == 0.0 finally: _restore_model_cost_entries(snapshot) -def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): - """Direct SDK calls (no router deployment id in metadata) keep the legacy - behavior: custom pricing is registered under ``{provider}/{model}`` and the - request is costed with it. - """ - model_key = "openai/lit3991-direct-sdk-embed-model" - override_input_cost = 3e-05 - try: - response = litellm.embedding( - model=model_key, - input=["hello"], - api_key="fake-key", - input_cost_per_token=override_input_cost, - output_cost_per_token=override_input_cost * 2, - mock_response=[0.1, 0.2], - ) +def _request_logger(model: str, call_type: str) -> Logging: + from datetime import datetime, timezone - assert ( - litellm.model_cost[model_key]["input_cost_per_token"] - == override_input_cost - ) - cost = litellm.completion_cost( - completion_response=response, - model=model_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - ) - assert cost == pytest.approx(10 * override_input_cost) - finally: - litellm.model_cost.pop(model_key, None) - _invalidate_model_cost_lowercase_map() + return Logging( + model=model, + messages=[], + stream=False, + call_type=call_type, + start_time=datetime.now(timezone.utc), + litellm_call_id="override", + function_id="override", + ) + + +def test_embedding_direct_sdk_custom_pricing_is_request_scoped() -> None: + model: Final = "openai/request-priced-embedding" + before: Final = _snapshot_model_cost_entries((model,)) + logger: Final = _request_logger(model, "embedding") + response: Final = litellm.embedding( + model=model, + input=["hello"], + api_key="fake-key", + input_cost_per_token=0.003, + output_cost_per_token=0.006, + mock_response=[0.1, 0.2], + litellm_logging_obj=logger, + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(10 * 0.003) + assert _snapshot_model_cost_entries((model,)) == before @pytest.mark.parametrize("route", ["", "converse/", "invoke/"]) @@ -1407,10 +1359,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing(): object stays untouched, and the caller's incoming nested dict is never inserted by reference into the merged result. """ - from litellm.utils import update_dictionary - from typing import cast + from litellm.utils import update_dictionary + existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"}) existing = {"off_peak_pricing": existing_nested} incoming_nested = cast( @@ -1609,22 +1561,24 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing(): del router -def test_completion_registers_cost_per_second_pricing(): - model_key: Final = "openai/test-cost-per-second-registration" - original_entries: Final = _snapshot_model_cost_entries([model_key]) +def test_completion_per_second_pricing_is_request_scoped() -> None: + from litellm.types.utils import ModelResponse, Usage - try: - litellm.completion( - model=model_key, - messages=[{"role": "user", "content": "hello"}], - api_key="fake-key", - cost_per_second=0.02, - mock_response="hello back", - ) - - assert litellm.model_cost[model_key]["cost_per_second"] == 0.02 - finally: - _restore_model_cost_entries(original_entries) + model: Final = "openai/request-priced-seconds" + logger: Final = _request_logger(model, "completion") + before: Final = _snapshot_model_cost_entries((model,)) + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + cost_per_second=0.02, + mock_response="hello back", + litellm_logging_obj=logger, + ) + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=0, completion_tokens=0)) + response._response_ms = 2000 + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(0.04) + assert _snapshot_model_cost_entries((model,)) == before def test_update_model_cost(): @@ -1676,31 +1630,139 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): _restore_model_cost_entries(model_cost_entries) -def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request +def test_per_request_custom_pricing_preserves_a_colliding_deployment() -> None: + from litellm.main import _set_request_pricing + from litellm.types.utils import ModelResponse, Usage - deployment_id: Final = "baseten/zai-org/glm-5.2" - builtin_key: Final = "baseten/zai-org/GLM-5.2" - shared_key: Final = "openai/zai-org/GLM-5.2" - model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key)) - builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) - openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) - try: - _register_custom_pricing_for_request( - model="zai-org/GLM-5.2", - custom_llm_provider="openai", - kwargs={ - "input_cost_per_token": 0.00000096, - "output_cost_per_token": 0.00000302, - "metadata": {"model_info": {"id": deployment_id}}, - }, - model_info={"mode": "chat"}, + deployment: Final = "baseten/zai-org/glm-5.2" + model: Final = "zai-org/GLM-5.2" + keys: Final = (deployment, "baseten/zai-org/GLM-5.2", f"openai/{model}") + before: Final = _snapshot_model_cost_entries(keys) + logger: Final = _request_logger(model, "completion") + logger.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) + _set_request_pricing( + logger, + model, + "openai", + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "metadata": {"model_info": {"id": deployment}}}, + {"mode": "chat"}, + ) + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5)) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(0.02) + assert _snapshot_model_cost_entries(keys) == before + + +@pytest.mark.parametrize("call_type", ("completion", "embedding")) +def test_concurrent_requests_keep_distinct_prices_and_leave_default_rates(call_type: str) -> None: + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + + model: Final = "openai/gpt-4o-mini" if call_type == "completion" else "openai/text-embedding-3-small" + before: Final = _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) + ready: Final = Barrier(2) + + def execute(logger: Logging, rate: float) -> object: + if call_type == "completion": + return litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=rate, + output_cost_per_token=rate, + mock_response="hello", + litellm_logging_obj=logger, + ) + return litellm.embedding( + model=model, + input=["hello"], + api_key="fake-key", + input_cost_per_token=rate, + output_cost_per_token=rate, + mock_response=[0.1], + litellm_logging_obj=logger, ) - entry: Final = litellm.model_cost[deployment_id] - assert entry["input_cost_per_token"] == 0.00000096 - assert entry["output_cost_per_token"] == 0.00000302 - assert litellm.model_cost[builtin_key] == builtin_row_before + def price(rate: float) -> float | None: + logger: Final = _request_logger(model, call_type) + response: Final = execute(logger, rate) + ready.wait(timeout=10) + return logger.response_cost_calculator(response, ignore_stamped_cost=True) + + with ThreadPoolExecutor(max_workers=2) as pool: + amounts: Final = tuple(pool.map(price, (0.001, 0.003))) + assert amounts[0] is not None and amounts[0] > 0 + assert amounts[1] == pytest.approx(amounts[0] * 3) + assert _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) == before + + +def test_request_pricing_preserves_reasoning_and_cache_rates() -> None: + from litellm.types.utils import ModelResponse, Usage + + model: Final = "openai/gpt-4o-mini" + logger: Final = _request_logger(model, "completion") + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=0.01, + output_cost_per_token=0.02, + cache_read_input_token_cost=0.001, + output_cost_per_reasoning_token=0.04, + mock_response="hello", + litellm_logging_obj=logger, + ) + response: Final = ModelResponse( + model="gpt-4o-mini", + usage=Usage( + prompt_tokens=100, + completion_tokens=20, + prompt_tokens_details={"cached_tokens": 40}, + completion_tokens_details={"reasoning_tokens": 5}, + ), + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx( + 60 * 0.01 + 40 * 0.001 + 15 * 0.02 + 5 * 0.04 + ) + + +@pytest.mark.parametrize("explicit_null_mode", (False, True)) +def test_router_custom_token_prices_accept_missing_deployment_mode(explicit_null_mode: bool) -> None: + from litellm import Router + from litellm.types.utils import ModelResponse + + deployment_id: Final = "custom-pricing-without-mode" + keys: Final = ("gpt-4o-mini", "openai/gpt-4o-mini", deployment_id) + original: Final = _snapshot_model_cost_entries(keys) + model_info: Final = {"id": deployment_id, **({"mode": None} if explicit_null_mode else {})} + try: + router: Final = Router( + model_list=[ + { + "model_name": "priced-chat", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "mock_response": "hello back", + }, + "model_info": model_info, + } + ], + num_retries=0, + ) + registered: Final = _snapshot_model_cost_entries(keys) + logger: Final = _request_logger("openai/gpt-4o-mini", "completion") + response: Final = router.completion( + model="priced-chat", messages=[{"role": "user", "content": "hello"}], litellm_logging_obj=logger + ) + assert isinstance(response, ModelResponse) + assert response.usage.prompt_tokens > 0 + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx( + response.usage.prompt_tokens * 0.001 + response.usage.completion_tokens * 0.002 + ) + assert _snapshot_model_cost_entries(keys) == registered finally: - _restore_model_cost_entries(model_cost_entries) - litellm.open_ai_chat_completion_models.intersection_update(openai_models_before) + _restore_model_cost_entries(original) From 47d53cad6de63c69d2af3eebf088eeba7a1ccc91 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 09:54:16 -0700 Subject: [PATCH 03/20] fix(accounting): avoid adjusting stamped streaming costs twice --- litellm/proxy/spend_tracking/cost_compiler.py | 4 +- .../spend_tracking/test_cost_compiler.py | 52 ++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 3580d2854e2..5a801039b50 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -136,7 +136,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: } } ], - usage=Usage.model_validate(captured.usage or {}), + usage=Usage.model_validate({name: value for name, value in (captured.usage or {}).items() if name != "cost"}), service_tier=response_inputs.get("service_tier"), meta={"billed_units": response_inputs.get("billed_units")}, hidden_params={ @@ -182,7 +182,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: custom_llm_provider=provider, litellm_params={**deployment, **request, "metadata": {"model_info": _fields(pricing.get("model_info"))}}, additional_response_cost=request.get("additional_response_cost"), - usage=Usage.model_validate(captured.usage or {}), + usage=Usage.model_validate({name: value for name, value in (captured.usage or {}).items() if name != "cost"}), end_time=event.ended_at, mcp_tool_call_metadata={ "name": _fields(pricing.get("tool")).get("name"), diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index bd093d6e012..f5e35ec7efc 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -149,8 +149,16 @@ def test_retained_token_pricing_preserves_cache_reasoning_and_service_tier() -> assert float(result.total_cost) == pytest.approx(60 * 0.03 + 40 * 0.002 + 15 * 0.05 + 5 * 0.07) -def test_upstream_reported_price_takes_precedence_without_custom_pricing() -> None: - result: Final = price_usage(json.dumps(_retained_envelope(logging={"provider_reported_cost": 0.037}))) +@pytest.mark.parametrize("stamped_cost", [None, 0.9]) +def test_upstream_reported_price_takes_precedence_without_custom_pricing(stamped_cost: float | None) -> None: + result: Final = price_usage( + json.dumps( + _retained_envelope( + logging={"provider_reported_cost": 0.037}, + usage={"prompt_tokens": 10, "completion_tokens": 4, "cost": stamped_cost}, + ) + ) + ) assert isinstance(result, PricingSnapshot) assert float(result.total_cost) == pytest.approx(0.037) @@ -847,3 +855,43 @@ def test_explicit_zero_retained_token_rates_are_known_free() -> None: assert isinstance(result, PricingSnapshot) assert result.total_cost == Decimal("0") assert result.pricing_source == "cost_map" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("completion_tokens", [1, 4]) +async def test_stream_replay_applies_adjustments_once_to_legacy_stamped_usage( + monkeypatch: pytest.MonkeyPatch, completion_tokens: int +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.05}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.1, "fixed_amount": 0.001}}) + retained: Final = _capture_logger().request_pricing + assert retained is not None + monkeypatch.setitem(litellm.model_cost, "openai/capture-model", retained.models()["capture-model"]) + monkeypatch.setitem(litellm.model_cost, "capture-model", retained.models()["capture-model"]) + logger: Final = _logger(stream=True) + response: Final = ModelResponse( + id="terminal", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=completion_tokens) + ) + expected: Final = _logger(stream=True).response_cost_calculator(response, ignore_stamped_cost=True) + assert expected is not None + stamped: Final = response.model_copy( + update={"usage": Usage(prompt_tokens=10, completion_tokens=completion_tokens, cost=expected)} + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, stamped, _AT, _AT) + await meter.close() + priced: Final = price_usage(database.rows[-1].envelope) + assert isinstance(priced, PricingSnapshot) + assert float(priced.total_cost) == pytest.approx(expected) + assert float(priced.original_cost or 0) == pytest.approx(0.1 + completion_tokens * 0.02) + finally: + await meter.stop(1) From a2b911d97b526b39d4bcb04bf7027015d496cb7b Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:17:00 -0700 Subject: [PATCH 04/20] fix(accounting): retain upstream usage prices during capture --- litellm/litellm_core_utils/litellm_logging.py | 2 ++ litellm/proxy/spend_tracking/cost_compiler.py | 22 +++++++++--- .../spend_tracking/test_cost_compiler.py | 35 +++++++++++++------ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 6d9df8a7ad5..e22ca540864 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -85,6 +85,7 @@ from litellm.litellm_core_utils.classifier_logging import ( is_classifier_call, ) from litellm.litellm_core_utils.core_helpers import ( + billed_usage_cost, finite_cost, get_provider_response_headers_from_hidden_params, is_expected_client_error, @@ -3076,6 +3077,7 @@ class Logging(LiteLLMLoggingBaseClass): { "response_ms": getattr(result, "_response_ms", None), "provider_reported_cost": self._provider_reported_cost, + "provider_usage_cost": billed_usage_cost(result, self.custom_llm_provider), "current_billing_time": self.billing_time_for_cost().isoformat(), "custom_pricing": self.custom_pricing_for(result), "custom_cost_per_token": self.custom_cost_per_token, diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 5a801039b50..37516add6d4 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -71,7 +71,11 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: ) provider: Final = next((d.value for d in event.dimensions if d.name is DimensionName.PROVIDER), None) logged: Final = _fields(pricing.get("logging")) - reported: Final = logged.get("provider_reported_cost") + reported: Final = ( + logged.get("provider_reported_cost") + if logged.get("provider_reported_cost") is not None + else logged.get("provider_usage_cost") + ) upstream: Final = ( isinstance(reported, (int, float)) and not isinstance(reported, bool) @@ -122,6 +126,12 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: ) response_inputs: Final = _fields(pricing.get("response_inputs")) tool_response: Final = _fields(pricing.get("tool_response")) + replay_usage: Final = Usage.model_validate( + { + **{name: value for name, value in (captured.usage or {}).items() if name != "cost"}, + **({"cost": reported} if upstream and logged.get("provider_reported_cost") is None else {}), + } + ) token_response: Final = ModelResponse( model=model, choices=[ @@ -136,7 +146,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: } } ], - usage=Usage.model_validate({name: value for name, value in (captured.usage or {}).items() if name != "cost"}), + usage=replay_usage, service_tier=response_inputs.get("service_tier"), meta={"billed_units": response_inputs.get("billed_units")}, hidden_params={ @@ -182,7 +192,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: custom_llm_provider=provider, litellm_params={**deployment, **request, "metadata": {"model_info": _fields(pricing.get("model_info"))}}, additional_response_cost=request.get("additional_response_cost"), - usage=Usage.model_validate({name: value for name, value in (captured.usage or {}).items() if name != "cost"}), + usage=replay_usage, end_time=event.ended_at, mcp_tool_call_metadata={ "name": _fields(pricing.get("tool")).get("name"), @@ -202,7 +212,9 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: } ) logger.record_provider_reported_cost( - TypeAdapter[float | None](float | None).validate_python(reported if upstream else None) + TypeAdapter[float | None](float | None).validate_python( + logged.get("provider_reported_cost") if upstream else None + ) ) billed_at: Final = TypeAdapter(datetime).validate_python(logged.get("current_billing_time", event.ended_at)) if event.call_type is CallTypes.arealtime and captured.usage and captured.usage.get("metered_usage") is not None: @@ -250,7 +262,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: "priced_model_key": key, "total_cost": Decimal(str(finalized.amount)), "guardrail_cost": Decimal(str(guardrail)), - "upstream_reported_cost": Decimal(str(logged["provider_reported_cost"])) if upstream else None, + "upstream_reported_cost": Decimal(str(reported)) if upstream else None, "additional_costs": tuple( {"name": name, "amount": Decimal(str(value))} for name, value in { diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index f5e35ec7efc..ea769816b59 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -347,25 +347,40 @@ async def test_real_agent_capture_prices_retained_per_query_fee() -> None: @pytest.mark.asyncio @pytest.mark.parametrize("reported_cost", (0.0, 0.123)) -async def test_real_capture_prices_upstream_cost_without_a_model_map(reported_cost: float) -> None: +@pytest.mark.parametrize("stamped", (False, True)) +@pytest.mark.parametrize("adjusted", (False, True)) +async def test_real_capture_prices_upstream_cost_without_a_model_map( + monkeypatch: pytest.MonkeyPatch, reported_cost: float, stamped: bool, adjusted: bool +) -> None: from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings from litellm.types.utils import ModelResponse, Usage from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + import litellm + from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage + + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.05} if adjusted else {}) + monkeypatch.setattr( + litellm, "cost_margin_config", {"global": {"percentage": 0.1, "fixed_amount": 0.001}} if adjusted else {} + ) + expected: Final = reported_cost * 0.95 * 1.1 + 0.001 if adjusted else reported_cost logger: Final = _logger() - logger.record_provider_reported_cost(reported_cost) + + response: Final = ModelResponse( + id="upstream", model="capture-model", usage=Usage(prompt_tokens=10, cost=reported_cost) + ) + if stamped: + stamp_litellm_bill_on_usage(response, response.usage, reported_cost + 0.9) database: Final = StoredUsage() meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) try: - capture_logging_usage( - meter, logger, ModelResponse(id="upstream", model="capture-model", usage=Usage(prompt_tokens=10)), _AT, _AT - ) + capture_logging_usage(meter, logger, response, _AT, _AT) await meter.close() result: Final = price_usage(database.rows[-1].envelope) assert isinstance(result, PricingSnapshot) - assert float(result.total_cost) == pytest.approx(reported_cost) + assert float(result.total_cost) == pytest.approx(expected) assert result.upstream_reported_cost == Decimal(str(reported_cost)) finally: await meter.stop(1) @@ -881,13 +896,13 @@ async def test_stream_replay_applies_adjustments_once_to_legacy_stamped_usage( ) expected: Final = _logger(stream=True).response_cost_calculator(response, ignore_stamped_cost=True) assert expected is not None - stamped: Final = response.model_copy( - update={"usage": Usage(prompt_tokens=10, completion_tokens=completion_tokens, cost=expected)} - ) + from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage + + stamp_litellm_bill_on_usage(response, response.usage, expected) database: Final = StoredUsage() meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) try: - capture_logging_usage(meter, logger, stamped, _AT, _AT) + capture_logging_usage(meter, logger, response, _AT, _AT) await meter.close() priced: Final = price_usage(database.rows[-1].envelope) assert isinstance(priced, PricingSnapshot) From da08f7c269d444a725b303ce42dd8905d19155eb Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:19:01 -0700 Subject: [PATCH 05/20] fix(accounting): preserve terminal usage pricing and charge ownership --- litellm/cost_calculator.py | 6 +- litellm/litellm_core_utils/litellm_logging.py | 5 +- litellm/proxy/spend_tracking/cost_compiler.py | 48 ++++- litellm/proxy/spend_tracking/usage_capture.py | 7 +- .../database/test_accounting_ledger.py | 14 +- .../spend_tracking/test_cost_compiler.py | 176 ++++++++++++++++++ .../spend_tracking/test_usage_capture.py | 31 +++ 7 files changed, 271 insertions(+), 16 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 76cced0c1c9..4a72e88ad3f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2368,15 +2368,15 @@ def completion_cost( def get_response_cost_from_hidden_params( - hidden_params: dict | BaseModel, + hidden_params: Mapping[str, object] | BaseModel, ) -> float | None: if isinstance(hidden_params, BaseModel): _hidden_params_dict = hidden_params.model_dump() else: _hidden_params_dict = hidden_params - additional_headers: Final = _hidden_params_dict.get("additional_headers", {}) - if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers: + additional_headers: Final = as_str_mapping(_hidden_params_dict.get("additional_headers")) + if additional_headers is not None and "llm_provider-x-litellm-response-cost" in additional_headers: return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"]) return None diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e22ca540864..c8f34995d06 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -61,6 +61,7 @@ from litellm.constants import ( from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, + get_response_cost_from_hidden_params, get_usage_object, pricing_entry_for_cost_calc, select_model_name_for_cost_calc, @@ -3076,7 +3077,9 @@ class Logging(LiteLLMLoggingBaseClass): return TypeAdapter(dict[str, JsonValue]).validate_python( { "response_ms": getattr(result, "_response_ms", None), - "provider_reported_cost": self._provider_reported_cost, + "provider_reported_cost": self._provider_reported_cost + if self._provider_reported_cost is not None + else get_response_cost_from_hidden_params(dict(get_hidden_params(result) or {})), "provider_usage_cost": billed_usage_cost(result, self.custom_llm_provider), "current_billing_time": self.billing_time_for_cost().isoformat(), "custom_pricing": self.custom_pricing_for(result), diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 37516add6d4..a0cca9c3757 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -9,7 +9,16 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter from litellm._internal_context import pinned_billing_time from litellm._logging import verbose_proxy_logger -from litellm.accounting import CostEvent, DeliveryTarget, DimensionName, OutboundUsageEvent, PricingSnapshot, Unpriced +from litellm.accounting import ( + CostEvent, + DeliveryTarget, + DimensionName, + OutboundUsageEvent, + Outcome, + PricingSnapshot, + Unpriced, +) +from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing @@ -18,6 +27,7 @@ from litellm.proxy.spend_tracking.usage_meter import usage_delivery_targets from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( CallTypes, + CostBreakdown, CostPerToken, LiteLLMRealtimeStreamLoggingObject, ModelResponse, @@ -71,6 +81,13 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: ) provider: Final = next((d.value for d in event.dimensions if d.name is DimensionName.PROVIDER), None) logged: Final = _fields(pricing.get("logging")) + if ( + fixed_call + and event.outcome is not Outcome.SUCCEEDED + and captured.usage is None + and not logged.get("tool_response_cost") + ): + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") reported: Final = ( logged.get("provider_reported_cost") if logged.get("provider_reported_cost") is not None @@ -180,7 +197,9 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: model=model, messages=[], stream=False, - call_type=event.call_type.value, + call_type=CallTypes.aresponses.value + if event.call_type in {CallTypes.get_responses, CallTypes.aget_responses} + else event.call_type.value, start_time=event.started_at, litellm_call_id=event.event_id, function_id=event.attempt_id, @@ -226,7 +245,26 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: ) logger.cost_billing_time = billed_at with retained_pricing(context), pinned_billing_time(billed_at): - amount: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + batch_costs: Final = ( + batch_cost_calculator( + replay_usage, + model, + provider, + data_residency=TypeAdapter[str | None](str | None).validate_python(deployment.get("data_residency")), + vertex_location=TypeAdapter[str | None](str | None).validate_python(deployment.get("vertex_location")), + ) + if event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} + else None + ) + amount: Final = ( + sum(batch_costs) + if batch_costs is not None + else logger.response_cost_calculator(response, ignore_stamped_cost=True) + ) + if batch_costs is not None: + logger.cost_breakdown = CostBreakdown( + input_cost=batch_costs[0], output_cost=batch_costs[1], total_cost=sum(batch_costs) + ) if amount is None: return Unpriced(usage_event_id=event.event_id, reason="calculation_failed") retained_final: Final = captured.finalized_cost or {} @@ -255,9 +293,9 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: "pricing_source": "upstream_header" if upstream else "per_request" - if request + if any(is_server_derived_pricing_key(name) and value is not None for name, value in request.items()) else "deployment" - if deployment + if any(is_server_derived_pricing_key(name) and value is not None for name, value in deployment.items()) else "cost_map", "priced_model_key": key, "total_cost": Decimal(str(finalized.amount)), diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index c1d87d752d8..674862713dc 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -307,7 +307,8 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res } deferred: Final = billed_id.endswith(("_batch_cost", "_background_cost")) and terminal management: Final = ( - (billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal) + payload["call_type"] in {CallTypes.create_batch.value, CallTypes.acreate_batch.value} + or (billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal) or (is_background_response(response) and not terminal) or is_unbilled_non_inference_call(payload["call_type"], request_metadata, response) ) @@ -836,8 +837,8 @@ def capture_logging_usage( existing_pricing: Final = details.get("_accounting_pricing_snapshot") pricing: Final = ( existing_pricing - if isinstance(existing_pricing, PricingCapture) and not internal - else _capture_pricing(logging_obj, normalized, details) + if isinstance(existing_pricing, PricingCapture) and not internal and not _batch_rows(normalized) + else _capture_pricing(logging_obj, normalized_result, details) ) if freeze_only or _batch_rows(normalized): if not internal: diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index f783a128bc7..efff0d9b16a 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -889,8 +889,9 @@ async def test_restarted_compiler_recovers_transient_failures_without_revisiting @pytest.mark.asyncio @pytest.mark.parametrize("sync_ack", (False, True)) +@pytest.mark.parametrize("provider_header", (False, True)) async def test_proxy_capture_and_background_compiler_converge_on_final_header_charge( - writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, sync_ack: bool + writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, sync_ack: bool, provider_header: bool ) -> None: import litellm from litellm.litellm_core_utils.litellm_logging import Logging @@ -929,12 +930,17 @@ async def test_proxy_capture_and_background_compiler_converge_on_final_header_ch logger.update_environment_variables( model="capture-priced-model", custom_llm_provider="openai", litellm_params={}, optional_params={} ) - logger.set_request_pricing( - "capture-priced-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} - ) + if not provider_header: + logger.set_request_pricing( + "capture-priced-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} + ) response: Final = ModelResponse( id="gateway-provider", model="capture-priced-model", usage=Usage(prompt_tokens=10, completion_tokens=4) ) + if provider_header: + from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params + + set_response_cost_in_hidden_params(response, 0.18) try: finalized: Final = await finalize_response_cost(logger, response, None, _AT, guardrail_cost=0.03) assert finalized is not None diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index ea769816b59..fe4e8ca7791 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -910,3 +910,179 @@ async def test_stream_replay_applies_adjustments_once_to_legacy_stamped_usage( assert float(priced.original_cost or 0) == pytest.approx(0.1 + completion_tokens * 0.02) finally: await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("amount", [0.0, 0.123]) +async def test_capture_retains_provider_header_price_without_catalog(amount: float) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + response: Final = ModelResponse( + model="capture-model", + usage=Usage(prompt_tokens=10, completion_tokens=4), + ) + from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params + + set_response_cost_in_hidden_params(response, amount) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, _logger(), response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert result.total_cost == Decimal(str(amount)) + assert result.pricing_source == "upstream_header" + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("metadata", [{"input_cost_per_token": None}, {"service_tier": "default"}]) +def test_catalog_provenance_ignores_metadata_without_price_overrides(metadata: dict[str, object]) -> None: + result: Final = price_usage(json.dumps(_retained_envelope(deployment=metadata, request=metadata))) + assert isinstance(result, PricingSnapshot) + assert result.pricing_source == "cost_map" + + +@pytest.mark.parametrize("call_type", ["get_responses", "aget_responses"]) +def test_terminal_response_replay_prices_usage_without_changing_event_identity(call_type: str) -> None: + from pydantic import TypeAdapter + + original: Final = _retained_envelope() + event: Final = TypeAdapter(dict[str, object]).validate_python(original["event"]) + result: Final = price_usage(json.dumps({**original, "event": {**event, "call_type": call_type}})) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.01 + 4 * 0.02) + + +@pytest.mark.asyncio +async def test_batch_terminal_capture_refreshes_row_prices_after_creation(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_batch_callback, capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + rates: Final = { + "mode": "chat", + "litellm_provider": "openai", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_token_batches": 0.003, + "output_cost_per_token_batches": 0.004, + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.1}}) + monkeypatch.setitem(litellm.model_cost, "second-model", {**rates, "input_cost_per_token_batches": 0.005}) + logger: Final = _logger("aretrieve_batch") + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + usage: Final = {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14} + response: Final = { + "id": "batch", + "status": "completed", + "usage": { + **usage, + "batch_usage_by_model": {"capture-model": usage, "second-model": usage}, + "batch_usage_by_request": [ + {"model": "capture-model", "usage": usage}, + {"model": "capture-model", "usage": usage}, + {"model": "second-model", "usage": usage}, + ], + }, + } + try: + capture_logging_usage(meter, logger, {"id": "batch", "status": "in_progress"}, _AT, _AT, freeze_only=True) + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + first_rows: Final = database.rows + monkeypatch.setitem(litellm.model_cost, "capture-model", {**rates, "input_cost_per_token_batches": 99.0}) + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + await meter.close() + assert database.rows == first_rows + results: Final = tuple(price_usage(row.envelope) for row in database.rows) + priced: Final = tuple(result for result in results if isinstance(result, PricingSnapshot)) + assert len(priced) == 3 + assert sorted(float(price.total_cost) for price in priced) == pytest.approx([0.046, 0.046, 0.066]) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["get_responses", "aget_responses"]) +async def test_background_capture_compiles_terminal_usage_once(call_type: str) -> None: + from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(call_type) + logger.update_environment_variables( + model="capture-model", + custom_llm_provider="openai", + optional_params={}, + litellm_params={"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}}, + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "background-job", + "object": "response", + "created_at": int(_AT.timestamp()), + "status": "completed", + "background": True, + "model": "capture-model", + "output": [], + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + } + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + assert len(database.rows) == 1 + assert database.rows[0].event.call_type.value == call_type + assert database.rows[0].event.event_id == "background-job_background_cost" + result: Final = price_usage(database.rows[0].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.01 + 4 * 0.02) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["call_mcp_tool", "asend_message"]) +async def test_failed_fixed_call_without_usage_does_not_invent_a_free_charge(call_type: str) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_callback + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(call_type) + logger.model_call_details["response_cost"] = 0.0 + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_callback( + meter, + {**logger.model_call_details, "litellm_logging_obj": logger, "log_event_type": "failed_api_call"}, + RuntimeError("Request rejected before dispatch"), + _AT, + _AT, + ) + await meter.close() + outbound: Final = database.rows[-1] + assert outbound.event.outcome is Outcome.FAILED + assert price_usage(outbound.envelope) == Unpriced( + usage_event_id=outbound.event.event_id, reason="usage_unknown" + ) + finally: + await meter.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index 24bb3480560..1ef6e6d2d4b 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1754,3 +1754,34 @@ async def test_sync_ack_deadline_bounds_compilation_after_usage_commit(monkeypat finally: compiler.release.set() await runtime.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["create_batch", "acreate_batch"]) +async def test_batch_creation_is_not_a_charge_owner(call_type: str) -> None: + from litellm.proxy.spend_tracking.cost_compiler import price_usage + from litellm.proxy.spend_tracking.usage_capture import capture_callback + from litellm.accounting import Unpriced + + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + captured: Final = capture_callback( + meter, + { + "call_type": call_type, + "litellm_call_id": "create", + "model": "capture-model", + "litellm_params": {}, + }, + {"id": "batch", "status": "validating"}, + NOW, + NOW, + ) + await meter.close() + outbound: Final = captured.writes[-1] + assert price_usage(outbound.envelope) == Unpriced( + usage_event_id=outbound.event.event_id, reason="not_charge_owner" + ) + finally: + await meter.stop(1) From 634bc134f5441c5f5d558d1f98a3f404202a05d9 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:23:32 -0700 Subject: [PATCH 06/20] fix(accounting): replay provider headers through shared pricing --- litellm/proxy/spend_tracking/cost_compiler.py | 8 +++----- tests/unit/proxy/spend_tracking/test_cost_compiler.py | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index a0cca9c3757..d85513da70a 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -19,6 +19,7 @@ from litellm.accounting import ( Unpriced, ) from litellm.cost_calculator import batch_cost_calculator +from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing @@ -230,11 +231,8 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: "file_search": adjustments.get("file_search") or None, } ) - logger.record_provider_reported_cost( - TypeAdapter[float | None](float | None).validate_python( - logged.get("provider_reported_cost") if upstream else None - ) - ) + if upstream and logged.get("provider_reported_cost") is not None: + set_hidden_param(response, "additional_headers", {"llm_provider-x-litellm-response-cost": reported}) billed_at: Final = TypeAdapter(datetime).validate_python(logged.get("current_billing_time", event.ended_at)) if event.call_type is CallTypes.arealtime and captured.usage and captured.usage.get("metered_usage") is not None: from litellm.constants import REALTIME_METERED_USAGE_KEY diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index fe4e8ca7791..c54f5bc7039 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -936,6 +936,7 @@ async def test_capture_retains_provider_header_price_without_catalog(amount: flo result: Final = price_usage(database.rows[-1].envelope) assert isinstance(result, PricingSnapshot) assert result.total_cost == Decimal(str(amount)) + assert (result.input_cost or Decimal(0)) + (result.output_cost or Decimal(0)) == result.total_cost assert result.pricing_source == "upstream_header" finally: await meter.stop(1) From 8c23bec98443129fd598792b844ff037d7d0b652 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:25:24 -0700 Subject: [PATCH 07/20] refactor(accounting): remove unused cost capture setter --- litellm/litellm_core_utils/litellm_logging.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c8f34995d06..41668370609 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3063,9 +3063,6 @@ class Logging(LiteLLMLoggingBaseClass): ) return normalized if assembled is None else assembled - def record_provider_reported_cost(self, cost: float | None) -> None: - self._provider_reported_cost = cost - def billing_time_for_cost(self) -> datetime.datetime: if self.cost_billing_time is None: self.cost_billing_time = current_billing_time() From e8b1f20be1a315588918a0df2ec7560207a759e9 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:15:03 -0700 Subject: [PATCH 08/20] fix(accounting): retain callback fees and terminal pricing context --- litellm/proxy/spend_tracking/cost_compiler.py | 11 ++- litellm/proxy/spend_tracking/usage_capture.py | 18 ++++- .../spend_tracking/test_cost_compiler.py | 81 +++++++++++++++++-- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index d85513da70a..e40ed19afcc 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -129,10 +129,17 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: for name, snapshot in _fields(pricing.get("realtime_models")).items() if _fields(snapshot).get("model_map_value") is not None } + base_pricing: Final = _fields(pricing.get("base_model_information")) + base_key: Final = base_pricing.get("model_map_key") context: Final = RetainedPricing( _json( { **secondary, + **( + {base_key: _fields(base_pricing.get("model_map_value"))} + if isinstance(base_key, str) and base_pricing.get("model_map_value") is not None + else {} + ), **{ alias: info for alias in dict.fromkeys((model, key, f"{provider}/{model}", model.removeprefix(f"{provider}/"))) @@ -267,7 +274,9 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: return Unpriced(usage_event_id=event.event_id, reason="calculation_failed") retained_final: Final = captured.finalized_cost or {} breakdown: Final = _fields(retained_final.get("cost_breakdown")) - guardrail: Final = TypeAdapter(float).validate_python(breakdown.get("guardrail_cost", 0.0)) + guardrail: Final = TypeAdapter(float).validate_python( + breakdown.get("guardrail_cost", pricing.get("guardrail_cost", 0.0)) + ) allowance: Final = TypeAdapter(float).validate_python(retained_final.get("allowance_discount", 0.0)) if allowance and captured.allocation is None: return Unpriced(usage_event_id=event.event_id, reason="allowance_allocation_missing") diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 674862713dc..5c735eaa6d9 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -281,6 +281,7 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res mask_api_base_credentials, ) from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost + from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation from litellm.proxy.spend_tracking.spend_tracking_utils import get_request_model_access_groups @@ -336,7 +337,11 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res else kwargs.get("_accounting_allocation") ) allocation: Final = allocation_value if isinstance(allocation_value, SearchAllowanceAllocation) else None - pricing_snapshot: Final = kwargs.get("_accounting_pricing_snapshot") + pricing_snapshot: Final = ( + _capture_pricing(logging_obj, response, kwargs) + if isinstance(logging_obj, Logging) and _batch_rows(response) + else kwargs.get("_accounting_pricing_snapshot") + ) observation: Final = kwargs.get("_accounting_usage_snapshot") captured_pricing: Final = { "model_map_information": ( @@ -345,6 +350,7 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res else metadata.get("model_map_information") ), "cost_breakdown": metadata.get("cost_breakdown"), + "guardrail_cost": guardrail_information_cost(metadata.get("guardrail_information")), **( _JSON_OBJECT.validate_json(pricing_snapshot.inputs) if isinstance(pricing_snapshot, PricingCapture) @@ -889,6 +895,7 @@ def _capture_effective_pricing( import litellm from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.utils import get_base_model_from_metadata params: Final = _fields(details.get("litellm_params")) prepared: Final = logging_obj.prepared_cost @@ -929,6 +936,7 @@ def _capture_effective_pricing( else None, } ) + base_model: Final = get_base_model_from_metadata(model_call_details=logging_obj.model_call_details) realtime_models: Final = tuple( dict.fromkeys( chain.from_iterable(_realtime_pricing_models(event) for event in _realtime_cost_events(normalized)) @@ -943,6 +951,14 @@ def _capture_effective_pricing( _json( { **_pricing_inputs(details, logging_obj, normalized), + "base_model_information": StandardLoggingPayloadSetup.get_model_cost_information( + base_model=base_model, + custom_pricing=False, + custom_llm_provider=None, + init_response_obj={"model": base_model}, + ) + if base_model is not None + else None, "realtime_models": { model: StandardLoggingPayloadSetup.get_model_cost_information( base_model=model, diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index c54f5bc7039..212b5c556ce 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -352,15 +352,14 @@ async def test_real_agent_capture_prices_retained_per_query_fee() -> None: async def test_real_capture_prices_upstream_cost_without_a_model_map( monkeypatch: pytest.MonkeyPatch, reported_cost: float, stamped: bool, adjusted: bool ) -> None: + import litellm + from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings from litellm.types.utils import ModelResponse, Usage from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush - import litellm - from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage - monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.05} if adjusted else {}) monkeypatch.setattr( litellm, "cost_margin_config", {"global": {"percentage": 0.1, "fixed_amount": 0.001}} if adjusted else {} @@ -914,7 +913,10 @@ async def test_stream_replay_applies_adjustments_once_to_legacy_stamped_usage( @pytest.mark.asyncio @pytest.mark.parametrize("amount", [0.0, 0.123]) -async def test_capture_retains_provider_header_price_without_catalog(amount: float) -> None: +@pytest.mark.parametrize("mapped", [False, True]) +async def test_capture_retains_provider_header_price_without_catalog( + amount: float, mapped: bool, monkeypatch: pytest.MonkeyPatch +) -> None: from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings from litellm.types.utils import ModelResponse, Usage @@ -928,14 +930,37 @@ async def test_capture_retains_provider_header_price_without_catalog(amount: flo from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params set_response_cost_in_hidden_params(response, amount) + import litellm + + logger: Final = _logger() + if mapped: + monkeypatch.setitem( + litellm.model_cost, + "mapped-cost-model", + { + "mode": "chat", + "litellm_provider": "openai", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + }, + ) + logger.model_call_details["litellm_params"]["metadata"] = {"model_info": {"base_model": "mapped-cost-model"}} + expected: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + expected_breakdown: Final = logger.cost_breakdown database: Final = StoredUsage() meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) try: - capture_logging_usage(meter, _logger(), response, _AT, _AT) + capture_logging_usage(meter, logger, response, _AT, _AT) await meter.close() + if mapped: + monkeypatch.setitem(litellm.model_cost, "mapped-cost-model", {"input_cost_per_token": 99.0}) result: Final = price_usage(database.rows[-1].envelope) assert isinstance(result, PricingSnapshot) assert result.total_cost == Decimal(str(amount)) + assert float(result.total_cost) == expected + assert expected_breakdown is not None + assert float(result.input_cost or 0) == pytest.approx(expected_breakdown["input_cost"]) + assert float(result.output_cost or 0) == pytest.approx(expected_breakdown["output_cost"]) assert (result.input_cost or Decimal(0)) + (result.output_cost or Decimal(0)) == result.total_cost assert result.pricing_source == "upstream_header" finally: @@ -961,7 +986,10 @@ def test_terminal_response_replay_prices_usage_without_changing_event_identity(c @pytest.mark.asyncio -async def test_batch_terminal_capture_refreshes_row_prices_after_creation(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("freeze_pricing", [False, True]) +async def test_batch_terminal_capture_refreshes_row_prices_after_creation( + monkeypatch: pytest.MonkeyPatch, freeze_pricing: bool +) -> None: import litellm from litellm.proxy.spend_tracking.usage_capture import capture_batch_callback, capture_logging_usage from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings @@ -998,8 +1026,11 @@ async def test_batch_terminal_capture_refreshes_row_prices_after_creation(monkey } try: capture_logging_usage(meter, logger, {"id": "batch", "status": "in_progress"}, _AT, _AT, freeze_only=True) - capture_logging_usage(meter, logger, response, _AT, _AT) - await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + if freeze_pricing: + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback( + meter, database.load, {**logger.model_call_details, "litellm_logging_obj": logger}, response, _AT, _AT + ) first_rows: Final = database.rows monkeypatch.setitem(litellm.model_cost, "capture-model", {**rates, "input_cost_per_token_batches": 99.0}) capture_logging_usage(meter, logger, response, _AT, _AT) @@ -1087,3 +1118,37 @@ async def test_failed_fixed_call_without_usage_does_not_invent_a_free_charge(cal ) finally: await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("billable", [False, True]) +async def test_capture_preserves_completed_guardrail_fee_without_header_finalization( + stream: bool, billable: bool +) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(stream=stream) + logger.model_call_details["litellm_params"]["metadata"] = { + "standard_logging_guardrail_information": [ + {"guardrail_name": "late-fee", "guardrail_cost": 0.03, "guardrail_cost_in_spend": billable} + ] + } + response: Final = ModelResponse( + id="guarded", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14) + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert result.guardrail_cost == Decimal("0.03" if billable else "0") + assert float(result.total_cost) == pytest.approx(0.18 + (0.03 if billable else 0)) + finally: + await meter.stop(1) From 1025640bddfd087b9f5b3765402f87d0960dbebc Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 12:33:28 -0700 Subject: [PATCH 09/20] fix(accounting): normalize unspecified retained pricing mode --- litellm/proxy/spend_tracking/cost_compiler.py | 2 +- .../spend_tracking/test_cost_compiler.py | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index e40ed19afcc..0ad72735401 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -108,7 +108,6 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: info: Final = { "key": key, "litellm_provider": provider, - "mode": "chat", "max_tokens": None, "max_input_tokens": None, "max_output_tokens": None, @@ -116,6 +115,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: "output_cost_per_token": None, "supported_openai_params": None, **rates, + "mode": rates.get("mode") or "chat", } adjustments: Final = _fields(pricing.get("adjustments")) discount: Final = adjustments.get("discount") diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 212b5c556ce..677dc2e4e08 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -913,9 +913,10 @@ async def test_stream_replay_applies_adjustments_once_to_legacy_stamped_usage( @pytest.mark.asyncio @pytest.mark.parametrize("amount", [0.0, 0.123]) -@pytest.mark.parametrize("mapped", [False, True]) +@pytest.mark.parametrize("mapped", [None, "mapped-cost-model", "openai/mapped-cost-model"]) +@pytest.mark.parametrize("provider", ["openai", "litellm_proxy"]) async def test_capture_retains_provider_header_price_without_catalog( - amount: float, mapped: bool, monkeypatch: pytest.MonkeyPatch + amount: float, mapped: str | None, provider: str, monkeypatch: pytest.MonkeyPatch ) -> None: from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings @@ -933,6 +934,9 @@ async def test_capture_retains_provider_header_price_without_catalog( import litellm logger: Final = _logger() + logger.update_environment_variables( + model="capture-model", custom_llm_provider=provider, litellm_params={}, optional_params={} + ) if mapped: monkeypatch.setitem( litellm.model_cost, @@ -944,7 +948,7 @@ async def test_capture_retains_provider_header_price_without_catalog( "output_cost_per_token": 0.02, }, ) - logger.model_call_details["litellm_params"]["metadata"] = {"model_info": {"base_model": "mapped-cost-model"}} + logger.model_call_details["litellm_params"]["metadata"] = {"model_info": {"base_model": mapped}} expected: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) expected_breakdown: Final = logger.cost_breakdown database: Final = StoredUsage() @@ -1152,3 +1156,14 @@ async def test_capture_preserves_completed_guardrail_fee_without_header_finaliza assert float(result.total_cost) == pytest.approx(0.18 + (0.03 if billable else 0)) finally: await meter.stop(1) + + +@pytest.mark.parametrize("mode", [None, "chat"]) +def test_header_split_retains_token_rates_when_catalog_mode_is_unspecified(mode: str | None) -> None: + result: Final = price_usage( + json.dumps(_retained_envelope(rates={"mode": mode}, logging={"provider_reported_cost": 0.18})) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.18) + assert float(result.input_cost or 0) == pytest.approx(0.1) + assert float(result.output_cost or 0) == pytest.approx(0.08) From ed4d3ae6ccc9f7582cf5e6404140791435575a8a Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:17:44 -0700 Subject: [PATCH 10/20] fix(accounting): enforce batch charge ownership and recover retained pricing --- .../migration.sql | 3 +- .../litellm_proxy_extras/schema.prisma | 1 + litellm/cost_calculator.py | 4 +- litellm/litellm_core_utils/litellm_logging.py | 8 +- litellm/proxy/db/accounting_ledger.py | 13 +- litellm/proxy/schema.prisma | 1 + litellm/proxy/spend_tracking/cost_compiler.py | 48 ++++- litellm/proxy/spend_tracking/usage_capture.py | 40 +++- schema.prisma | 1 + .../database/test_accounting_ledger.py | 146 +++++++++++++- .../spend_tracking/test_cost_compiler.py | 181 ++++++++++++++++++ .../spend_tracking/test_usage_capture.py | 11 +- 12 files changed, 427 insertions(+), 30 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql index 04fbdfdc237..ef18da183ed 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql @@ -1,5 +1,6 @@ ALTER TABLE "LiteLLM_AccountingEvent" ADD COLUMN "compilation_state" TEXT NOT NULL DEFAULT 'pending', -ADD COLUMN "compilation_error" TEXT; +ADD COLUMN "compilation_error" TEXT, +ADD COLUMN "compilation_retry_at" TIMESTAMP(3); CREATE INDEX "LiteLLM_AccountingEvent_kind_compilation_state_recorded_at_idx" ON "LiteLLM_AccountingEvent"("kind", "compilation_state", "recorded_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 048e0e1e17a..45ea5cd20d0 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2113,6 +2113,7 @@ model LiteLLM_AccountingEvent { payload Json compilation_state String @default("pending") compilation_error String? + compilation_retry_at DateTime? @@index([recorded_at]) @@index([kind, source_event_id]) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4a72e88ad3f..d50627cceb0 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2009,7 +2009,7 @@ def completion_cost( video_resolution=video_resolution, ) elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = _speech_billed_characters( + prompt_characters = speech_billed_characters( completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, prompt, ) @@ -2527,7 +2527,7 @@ def response_cost_calculator( raise e -def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: +def speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 41668370609..c87aaabc4b7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -65,6 +65,7 @@ from litellm.cost_calculator import ( get_usage_object, pricing_entry_for_cost_calc, select_model_name_for_cost_calc, + speech_billed_characters, ) from litellm.exceptions import ( BudgetExceededError, @@ -3069,8 +3070,6 @@ class Logging(LiteLLMLoggingBaseClass): return self.cost_billing_time def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]: - from litellm.utils import count_characters - return TypeAdapter(dict[str, JsonValue]).validate_python( { "response_ms": getattr(result, "_response_ms", None), @@ -3086,7 +3085,10 @@ class Logging(LiteLLMLoggingBaseClass): if self.model_call_details.get("call_type") in (CallTypes.call_mcp_tool.value, CallTypes.send_message.value, CallTypes.asend_message.value) else None, - "prompt_characters": count_characters(self._prompt_for_cost_calculation()) + "prompt_characters": speech_billed_characters( + result if isinstance(result, HttpxBinaryResponseContent) else None, + self._prompt_for_cost_calculation(), + ) if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value) else None, } diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index b479cea6a0d..7ebdfb70506 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -164,14 +164,22 @@ FROM "LiteLLM_AccountingEvent" WHERE source_event_id = $1 AND kind = 'cost' _PENDING_COMPILATION: Final = """ SELECT payload::text AS payload FROM "LiteLLM_AccountingEvent" WHERE kind = 'outbound_usage' AND compilation_state = 'pending' +AND (compilation_retry_at IS NULL OR compilation_retry_at <= CURRENT_TIMESTAMP) ORDER BY recorded_at, event_id LIMIT $1::int """ _COMPILATION_RESULT: Final = """ -UPDATE "LiteLLM_AccountingEvent" SET compilation_state = $2, compilation_error = $3 +UPDATE "LiteLLM_AccountingEvent" SET compilation_state = $2, compilation_error = $3, compilation_retry_at = NULL WHERE event_id = $1 AND kind = 'outbound_usage' """ +_RETRY_COMPILATION: Final = """ +UPDATE "LiteLLM_AccountingEvent" +SET compilation_error = 'calculation_retryable', compilation_retry_at = CURRENT_TIMESTAMP + INTERVAL '30 seconds' +WHERE event_id = $1 AND kind = 'outbound_usage' AND compilation_state = 'pending' +""" + + async def _unpriced(db: Prisma, event_id: str, reason: str) -> Unpriced: await db.execute_raw( _COMPILATION_RESULT, event_id, "excluded" if reason == "not_charge_owner" else "unpriced", reason @@ -401,6 +409,9 @@ class AccountingLedger: priced: Final = price(source["payload"]) except ValueError: return await _unpriced(db, validated.event_id, "invalid_pricing_inputs") + except Exception: # noqa: BLE001 # retain failed pricing for bounded retries without starving newer events + await db.execute_raw(_RETRY_COMPILATION, validated.event_id) + return Unpriced(usage_event_id=validated.event_id, reason="calculation_retryable") if isinstance(priced, Unpriced): if priced.usage_event_id != validated.event_id: raise AccountingConflictError("Unpriced result does not match its source usage") diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 048e0e1e17a..45ea5cd20d0 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2113,6 +2113,7 @@ model LiteLLM_AccountingEvent { payload Json compilation_state String @default("pending") compilation_error String? + compilation_retry_at DateTime? @@index([recorded_at]) @@index([kind, source_event_id]) diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 0ad72735401..52452a79106 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -5,6 +5,7 @@ from datetime import datetime from decimal import Decimal from typing import Final +from httpx import Response from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter from litellm._internal_context import pinned_billing_time @@ -18,19 +19,21 @@ from litellm.accounting import ( PricingSnapshot, Unpriced, ) -from litellm.cost_calculator import batch_cost_calculator +from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing +from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo from litellm.proxy.db.accounting_ledger import AccountingLedger from litellm.proxy.spend_tracking.usage_meter import usage_delivery_targets -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.utils import ( CallTypes, CostBreakdown, CostPerToken, LiteLLMRealtimeStreamLoggingObject, + ModelInfo, ModelResponse, StandardBuiltInToolsParams, Usage, @@ -55,6 +58,7 @@ class _UsageEnvelope(BaseModel): usage: Mapping[str, JsonValue] | None = None pricing: Mapping[str, JsonValue] charge_eligible: bool + parent_request_id: str | None = None finalized_cost: Mapping[str, JsonValue] | None = None allocation: Mapping[str, JsonValue] | None = None @@ -62,8 +66,19 @@ class _UsageEnvelope(BaseModel): def price_usage(envelope: str) -> PricingSnapshot | Unpriced: captured: Final = _UsageEnvelope.model_validate_json(envelope) event: Final = captured.event - if not captured.charge_eligible: + batch_summary: Final = event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} and not ( + captured.parent_request_id is not None and event.event_id.startswith(f"{captured.parent_request_id}:row:") + ) + if not captured.charge_eligible or batch_summary: return Unpriced(usage_event_id=event.event_id, reason="not_charge_owner") + logged: Final = _fields(captured.pricing.get("logging")) + speech_characters: Final = logged.get("prompt_characters") + speech_usage: Final = ( + event.call_type in {CallTypes.speech, CallTypes.aspeech} + and isinstance(speech_characters, int) + and not isinstance(speech_characters, bool) + and speech_characters >= 0 + ) fixed_call: Final = event.call_type in {CallTypes.call_mcp_tool, CallTypes.send_message, CallTypes.asend_message} realtime_events: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python( captured.pricing.get("realtime_events") or () @@ -71,17 +86,22 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: realtime_usage: Final = event.call_type is CallTypes.arealtime and any( bool(item.get("usage")) for item in realtime_events ) - if captured.usage is None and not fixed_call and not realtime_usage: + if captured.usage is None and not fixed_call and not realtime_usage and not speech_usage: return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") pricing: Final = captured.pricing mapped: Final = _fields(pricing.get("model_map_information")) rates: Final = _fields(mapped.get("model_map_value")) + if ( + event.call_type in {CallTypes.speech, CallTypes.aspeech} + and rates.get("input_cost_per_character") is not None + and not speech_usage + ): + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") model: Final = next( (d.value for d in event.dimensions if d.name is DimensionName.MODEL), event.call_type.value if fixed_call else None, ) provider: Final = next((d.value for d in event.dimensions if d.name is DimensionName.PROVIDER), None) - logged: Final = _fields(pricing.get("logging")) if ( fixed_call and event.outcome is not Outcome.SUCCEEDED @@ -180,8 +200,14 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: }, response_ms=logged.get("response_ms"), ) + speech_response: Final = HttpxBinaryResponseContent(Response(200, content=b"")) if speech_usage else None + if speech_response is not None: + set_hidden_param(speech_response, "billed_characters", speech_characters) + speech_response.usage = replay_usage response: Final = ( - LiteLLMRealtimeStreamLoggingObject.model_validate( + speech_response + if speech_response is not None + else LiteLLMRealtimeStreamLoggingObject.model_validate( { "results": pricing.get("realtime_events", []), "usage": captured.usage or {}, @@ -251,7 +277,15 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: logger.cost_billing_time = billed_at with retained_pricing(context), pinned_billing_time(billed_at): batch_costs: Final = ( - batch_cost_calculator( + ocr_batch_cost( + model=model, + custom_llm_provider=provider, + usage_info=OCRUsageInfo.model_validate(captured.usage), + model_info=TypeAdapter(ModelInfo).validate_python(info), + ) + if event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} + and any(name in (captured.usage or {}) for name in OCRUsageInfo.model_fields) + else batch_cost_calculator( replay_usage, model, provider, diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 5c735eaa6d9..4fede891018 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -173,7 +173,15 @@ def usage_writes( ended_at=ended, outcome=outcome, dimensions=dimensions, - metrics=_usage_metrics(None if context.usage_source == "unknown" else usage), + metrics=_usage_metrics( + None + if context.usage_source == "unknown" + or ( + context.ownership == "aggregate" + and payload["call_type"] in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value} + ) + else usage + ), ) envelope: Final = { "event": outbound.model_dump(mode="json"), @@ -218,6 +226,7 @@ def usage_writes( _INCLUDED_IN_PARENT: Final = object() +_BATCH_ROWS: Final = object() _PRICING_FIELDS: Final = frozenset( @@ -306,6 +315,10 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res "expired", "incomplete", } + batch_summary: Final = ( + payload["call_type"] in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value} + and kwargs.get("_accounting_batch_rows") is not _BATCH_ROWS + ) deferred: Final = billed_id.endswith(("_batch_cost", "_background_cost")) and terminal management: Final = ( payload["call_type"] in {CallTypes.create_batch.value, CallTypes.acreate_batch.value} @@ -320,6 +333,8 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res if management else f"{billed_id}:included" if included_in_parent + else f"{billed_id}:summary" + if batch_summary and terminal else billed_id ) if not attempt or attempt == "None" or (cached and not caller): @@ -390,12 +405,12 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res parent_request_id=parent, retry_of_attempt_id=_text(request_metadata.get("retry_of_attempt_id")), job_id=_text(raw_response.get("id")) if deferred or management else None, - charge_eligible=not cached and not included_in_parent and not management, + charge_eligible=not cached and not included_in_parent and not management and not batch_summary, request_count_eligible=bool(caller) and not origin and not retry and not deferred and not included_in_parent, ownership="included_in_parent" if included_in_parent else "aggregate" - if kwargs.get("additional_response_cost") is not None + if batch_summary or kwargs.get("additional_response_cost") is not None else "attempt", pricing=_json(captured_pricing), finalized_cost=_json( @@ -564,13 +579,26 @@ def _batch_rows(response: object) -> tuple[Mapping[str, JsonValue], ...]: def _batch_root(write: UsageWrite, rows: tuple[Mapping[str, JsonValue], ...]) -> UsageWrite: - event: Final = write.event.model_copy(update={"metrics": None}) + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_batch_id_from_unified_batch_id, + get_original_file_id, + ) + envelope: Final = _JSON_OBJECT.validate_json(write.envelope) + observed_job: Final = TypeAdapter(str).validate_python(envelope["job_id"]) + decoded: Final = convert_b64_uid_to_unified_uid(observed_job) + job: Final = ( + get_batch_id_from_unified_batch_id(decoded) if decoded != observed_job else get_original_file_id(observed_job) + ) + identity: Final = f"{job}_batch_cost" + event: Final = write.event.model_copy(update={"metrics": None, "event_id": identity, "attempt_id": identity}) return _database_usage_write( event, { **envelope, "event": event.model_dump(mode="json"), + "job_id": job, "usage": { "batch_row_count": len(rows), "batch_rows_sha256": sha256(_json(rows).encode()).hexdigest(), @@ -644,7 +672,7 @@ def _prepare_batch_capture( from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload rows: Final = tuple(sorted(_batch_rows(response), key=_json)) - builder: Final = _CaptureBuilder(kwargs, response) + builder: Final = _CaptureBuilder({**kwargs, "_accounting_batch_rows": _BATCH_ROWS}, response) payload: Final = get_logging_payload( dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe ) @@ -846,7 +874,7 @@ def capture_logging_usage( if isinstance(existing_pricing, PricingCapture) and not internal and not _batch_rows(normalized) else _capture_pricing(logging_obj, normalized_result, details) ) - if freeze_only or _batch_rows(normalized): + if freeze_only or logging_obj.call_type in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value}: if not internal: logging_obj.record_usage_pricing(pricing) return diff --git a/schema.prisma b/schema.prisma index 048e0e1e17a..45ea5cd20d0 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2113,6 +2113,7 @@ model LiteLLM_AccountingEvent { payload Json compilation_state String @default("pending") compilation_error String? + compilation_retry_at DateTime? @@index([recorded_at]) @@index([kind, source_event_id]) diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index efff0d9b16a..8703559d6aa 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -358,7 +358,9 @@ async def test_compilation_rejects_changed_source_before_reusing_a_charge(writer @pytest.mark.asyncio -async def test_compilation_failure_rolls_back_and_can_be_retried(writer: _Writer, ledger_database: str) -> None: +async def test_compilation_failure_retains_retry_without_a_partial_charge( + writer: _Writer, ledger_database: str +) -> None: _, store = writer event: Final = _usage() assert isinstance(event, OutboundUsageEvent) @@ -367,8 +369,9 @@ async def test_compilation_failure_rolls_back_and_can_be_retried(writer: _Writer def failed_pricing(envelope: str) -> PricingSnapshot: raise RuntimeError("pricing temporarily unavailable") - with pytest.raises(RuntimeError, match="pricing temporarily unavailable"): - await LoggingCostCompiler(store, failed_pricing, _TARGETS).compile(event) + assert await LoggingCostCompiler(store, failed_pricing, _TARGETS).compile(event) == Unpriced( + usage_event_id=event.event_id, reason="calculation_retryable" + ) assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] result: Final = await LoggingCostCompiler(store, lambda _: _cost(amount="0").snapshot, _TARGETS).compile(event) assert isinstance(result, CostEvent) @@ -872,12 +875,19 @@ async def test_restarted_compiler_recovers_transient_failures_without_revisiting return _cost().snapshot await LoggingCostCompiler(store, transient, _TARGETS).compile_pending() - assert await store.pending_compilation() == (delayed,) + assert await store.pending_compilation() == () + assert _rows( + ledger_database, + "SELECT compilation_state, compilation_error FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'delayed'", + ) == [("pending", "calculation_retryable")] assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ (healthy.event_id,) ] async with connected_proxy_prisma(ledger_database) as restarted: recovered: Final = AccountingLedger.for_client(restarted) + await restarted.db.execute_raw( + 'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = NULL WHERE event_id = $1', delayed.event_id + ) await LoggingCostCompiler(recovered, lambda _: _cost().snapshot, _TARGETS).compile_pending() assert await recovered.pending_compilation() == () await LoggingCostCompiler(recovered, transient, _TARGETS).compile_pending() @@ -982,3 +992,131 @@ async def test_proxy_capture_and_background_compiler_converge_on_final_header_ch finally: release.set() await runtime.stop(5) + + +@pytest.mark.asyncio +async def test_failing_compilation_page_does_not_starve_later_usage(writer: _Writer, ledger_database: str) -> None: + _, store = writer + broken: Final = _usage("broken-price") + healthy: Final = _usage("healthy-price") + assert isinstance(broken, OutboundUsageEvent) + assert isinstance(healthy, OutboundUsageEvent) + await _insert(store, broken) + await _insert(store, healthy) + + def price(envelope: str) -> PricingSnapshot: + if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == broken.event_id: + raise RuntimeError("temporary calculator failure") + return _cost().snapshot + + await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) + await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) + assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ + (healthy.event_id,) + ] + assert isinstance(await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(broken), CostEvent) + assert await store.pending_compilation() == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("summary_first", (False, True)) +async def test_batch_callbacks_commit_only_row_charges_across_restart( + writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, summary_first: bool +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import ( + UsageBatchWriter, + capture_batch_callback, + capture_callback, + capture_logging_usage, + ) + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + _, store = writer + rates: Final = { + "input_cost_per_token_batches": 0.003, + "output_cost_per_token_batches": 0.004, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + usage: Final = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} + response: Final = { + "id": "managed-job", + "status": "completed", + "usage": { + "batch_usage_by_model": {"capture-model": usage}, + "batch_usage_by_request": [{"model": "capture-model", "usage": usage} for _ in range(2)], + }, + } + logger: Final = _logger("aretrieve_batch") + batches: Final = UsageBatchWriter(store.insert_usage_batch, store.load_usage_batch) + meter: Final = DurableUsageMeter(batches.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) + compiler: Final = LoggingCostCompiler(store) + capture_logging_usage(meter, logger, response, _AT, _AT, freeze_only=True) + kwargs: Final = {**logger.model_call_details, "litellm_logging_obj": logger} + + async def summary() -> None: + capture_callback( + meter, + kwargs, + { + "id": "provider-job", + "status": "completed", + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + _AT, + _AT, + ) + await meter.flush() + await compiler.compile_pending() + + try: + if summary_first: + await summary() + await capture_batch_callback(meter, store.load_usage_batch, kwargs, response, _AT, _AT) + await compiler.compile_pending() + if not summary_first: + await summary() + before: Final = _rows( + ledger_database, 'SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" ORDER BY event_id' + ) + costs: Final = _rows( + ledger_database, "SELECT payload::text FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'" + ) + deliveries: Final = _rows( + ledger_database, + 'SELECT event_id, target, state FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target', + ) + assert len(costs) == 2 + snapshots: Final = tuple( + CostEvent.model_validate_json(json.dumps(json.loads(str(row[0]))["event"])).snapshot for row in costs + ) + assert sum(float(snapshot.total_cost) for snapshot in snapshots) == pytest.approx(2 * (10 * 0.003 + 5 * 0.004)) + monkeypatch.setitem(litellm.model_cost, "capture-model", {**rates, "input_cost_per_token_batches": 99.0}) + async with connected_proxy_prisma(ledger_database) as restarted: + ledger: Final = AccountingLedger.for_client(restarted) + from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model + + for job_id in ("managed-job", encode_file_id_with_model("managed-job", "capture-model", "batch")): + await capture_batch_callback( + meter, ledger.load_usage_batch, kwargs, {**response, "id": job_id}, _AT, _AT + ) + await asyncio.gather(*(LoggingCostCompiler(ledger).compile_pending() for _ in range(2))) + assert ( + _rows(ledger_database, 'SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" ORDER BY event_id') + == before + ) + assert ( + _rows( + ledger_database, + 'SELECT event_id, target, state FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target', + ) + == deliveries + ) + for row, snapshot in zip(costs, snapshots, strict=True): + assert price_usage(json.dumps(json.loads(str(row[0]))["pricing_inputs"])) == snapshot + finally: + await meter.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 677dc2e4e08..1b89f472248 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1167,3 +1167,184 @@ def test_header_split_retains_token_rates_when_catalog_mode_is_unspecified(mode: assert float(result.total_cost) == pytest.approx(0.18) assert float(result.input_cost or 0) == pytest.approx(0.1) assert float(result.output_cost or 0) == pytest.approx(0.08) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("logging_first", (False, True)) +async def test_batch_logging_and_rows_have_one_charge_owner( + monkeypatch: pytest.MonkeyPatch, logging_first: bool +) -> None: + import litellm + from litellm.batches.batch_utils import batch_cost_and_usage + from litellm.proxy.spend_tracking.usage_capture import ( + UsageBatchWriter, + capture_batch_callback, + capture_callback, + capture_logging_usage, + ) + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import LiteLLMBatch + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + rates: Final = { + "input_cost_per_token_batches": 0.003, + "output_cost_per_token_batches": 0.004, + "input_cost_per_token": 0.006, + "output_cost_per_token": 0.008, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + billed: Final = batch_cost_and_usage( + [ + { + "response": { + "status_code": 200, + "body": { + "model": "capture-model", + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + } + } + for _ in range(2) + ], + "openai", + ) + response: Final = LiteLLMBatch( + id="owner-job", + object="batch", + endpoint="/v1/chat/completions", + input_file_id="file", + completion_window="24h", + status="completed", + created_at=1, + completed_at=2, + usage=billed.usage, + ) + database: Final = StoredUsage() + writer: Final = UsageBatchWriter(database.insert, database.load) + meter: Final = DurableUsageMeter(writer.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) + logger: Final = _logger("aretrieve_batch") + capture_logging_usage(meter, logger, response, _AT, _AT, freeze_only=True) + try: + for _ in range(2): + if logging_first: + capture_callback( + meter, + {**logger.model_call_details, "litellm_logging_obj": logger}, + { + **response.model_dump(), + "id": "provider-job", + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + _AT, + _AT, + ) + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback( + meter, database.load, {**logger.model_call_details, "litellm_logging_obj": logger}, response, _AT, _AT + ) + if not logging_first: + capture_callback( + meter, + {**logger.model_call_details, "litellm_logging_obj": logger}, + { + **response.model_dump(), + "id": "provider-job", + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + _AT, + _AT, + ) + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.flush() + costs: Final = tuple( + price_usage(row.envelope) for row in database.rows if isinstance(row.event, OutboundUsageEvent) + ) + priced: Final = tuple(cost for cost in costs if isinstance(cost, PricingSnapshot)) + assert len(priced) == 2, costs + assert sum(float(cost.total_cost) for cost in priced) == pytest.approx(2 * (10 * 0.003 + 5 * 0.004)) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("characters", (0, 17)) +def test_speech_replay_prices_retained_characters(characters: int) -> None: + original: Final = _retained_envelope( + rates={"input_cost_per_character": 0.003, "mode": "audio_speech"}, logging={"prompt_characters": characters} + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={"call_type": CallTypes.aspeech} + ) + result: Final = price_usage(json.dumps({**original, "event": event.model_dump(mode="json")})) + assert isinstance(result, PricingSnapshot), result + assert float(result.total_cost) == pytest.approx(characters * 0.003) + + +def test_batch_ocr_replay_prices_pages_and_annotations() -> None: + original: Final = _retained_envelope( + rates={"ocr_cost_per_page_batches": 0.003, "annotation_cost_per_page_batches": 0.004, "mode": "ocr"}, + usage={"pages_processed": 3, "pages_processed_annotation": 2}, + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={"call_type": CallTypes.aretrieve_batch} + ) + result: Final = price_usage( + json.dumps( + { + **original, + "event": {**event.model_dump(mode="json"), "event_id": "batch:row:0"}, + "parent_request_id": "batch", + } + ) + ) + assert isinstance(result, PricingSnapshot), result + assert float(result.total_cost) == pytest.approx(3 * 0.003 + 2 * 0.004) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reported_characters", (None, 5)) +async def test_speech_capture_replays_billed_quantities_without_input_text(reported_characters: int | None) -> None: + from httpx import Response + + from litellm.litellm_core_utils.hidden_params import set_hidden_param + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import HttpxBinaryResponseContent + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger("aspeech") + logger.set_request_pricing("capture-model", "openai", {"input_cost_per_character": 0.003, "mode": "audio_speech"}) + logger.model_call_details["input"] = "privatespeechinput" + response: Final = HttpxBinaryResponseContent(Response(200, content=b"audio")) + if reported_characters is not None: + set_hidden_param(response, "billed_characters", reported_characters) + expected: Final = (reported_characters if reported_characters is not None else len("privatespeechinput")) * 0.003 + assert logger.response_cost_calculator(response) == pytest.approx(expected) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.flush() + envelope: Final = next(row.envelope for row in database.rows if isinstance(row.event, OutboundUsageEvent)) + assert "privatespeechinput" not in envelope + priced: Final = price_usage(envelope) + assert isinstance(priced, PricingSnapshot), priced + assert float(priced.total_cost) == pytest.approx(expected) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("characters", (None, -1, True)) +def test_character_pricing_does_not_invent_zero_for_missing_or_invalid_quantity(characters: object) -> None: + original: Final = _retained_envelope( + rates={"input_cost_per_character": 0.003}, logging={"prompt_characters": characters} + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={"call_type": CallTypes.aspeech} + ) + assert price_usage(json.dumps({**original, "event": event.model_dump(mode="json")})) == Unpriced( + usage_event_id=event.event_id, reason="usage_unknown" + ) diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index 1ef6e6d2d4b..c83d11250c9 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -334,7 +334,7 @@ async def test_competing_writers_reuse_only_terminal_pricing_snapshots(terminal: @pytest.mark.asyncio -async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_chargeable() -> None: +async def test_batch_poll_and_terminal_summary_are_not_charge_owners() -> None: from litellm.proxy.spend_tracking.usage_capture import capture_callback database: Final = StoredUsage() @@ -352,8 +352,8 @@ async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_charg await meter.close() assert pending.writes[-1].event.event_id == "poll:status" assert json.loads(pending.writes[-1].envelope)["charge_eligible"] is False - assert completed.writes[-1].event.event_id == "batch_batch_cost" - assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is True + assert completed.writes[-1].event.event_id == "batch_batch_cost:summary" + assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is False assert len(completed.writes) == 1 @@ -1021,9 +1021,8 @@ async def test_batch_capture_provenance_tracks_observations_not_envelope_presenc envelope: Final = json.loads(database.rows[-1].envelope) assert envelope["usage_source"] == ("unknown" if measured is None else "measured") assert envelope["usage"]["batch_usage_by_model"] == usage["batch_usage_by_model"] - assert database.rows[-1].event.metrics == ( - None if measured is None else (Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(measured["total_tokens"])),) - ) + assert database.rows[-1].event.metrics is None + assert envelope["charge_eligible"] is False @pytest.mark.asyncio From 2f4fa0399d786b1583925c40200ff08bd1246724 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:21:27 -0700 Subject: [PATCH 11/20] fix(accounting): prioritize unattempted usage before compilation retries --- litellm/proxy/db/accounting_ledger.py | 2 +- tests/integration/database/test_accounting_ledger.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 7ebdfb70506..0dbc3d8c86a 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -165,7 +165,7 @@ _PENDING_COMPILATION: Final = """ SELECT payload::text AS payload FROM "LiteLLM_AccountingEvent" WHERE kind = 'outbound_usage' AND compilation_state = 'pending' AND (compilation_retry_at IS NULL OR compilation_retry_at <= CURRENT_TIMESTAMP) -ORDER BY recorded_at, event_id LIMIT $1::int +ORDER BY compilation_retry_at NULLS FIRST, recorded_at, event_id LIMIT $1::int """ _COMPILATION_RESULT: Final = """ UPDATE "LiteLLM_AccountingEvent" SET compilation_state = $2, compilation_error = $3, compilation_retry_at = NULL diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 8703559d6aa..0171d3683ca 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -1010,6 +1010,11 @@ async def test_failing_compilation_page_does_not_starve_later_usage(writer: _Wri return _cost().snapshot await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) + _rows( + ledger_database, + 'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = CURRENT_TIMESTAMP WHERE event_id = %s', + (broken.event_id,), + ) await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ (healthy.event_id,) From 762ce2435560a8fa69630f830468eb258640ed38 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:05:53 -0700 Subject: [PATCH 12/20] fix(accounting): preserve pricing semantics and retry failed compilation --- .../migration.sql | 8 +- .../litellm_proxy_extras/schema.prisma | 2 +- litellm/cost_calculator.py | 81 ++++--- litellm/litellm_core_utils/litellm_logging.py | 18 +- .../llm_cost_calc/pricing_context.py | 17 +- litellm/proxy/db/accounting_ledger.py | 8 +- litellm/proxy/schema.prisma | 2 +- litellm/proxy/spend_tracking/cost_compiler.py | 33 ++- litellm/proxy/spend_tracking/usage_capture.py | 120 +++++++-- litellm/types/utils.py | 1 + schema.prisma | 2 +- .../database/test_accounting_ledger.py | 10 +- .../spend_tracking/test_cost_compiler.py | 227 ++++++++++++++++++ .../spend_tracking/test_usage_capture.py | 13 +- tests/unit/test_utils.py | 6 +- 15 files changed, 461 insertions(+), 87 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql index ef18da183ed..5bdedb25c25 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql @@ -1,6 +1,6 @@ ALTER TABLE "LiteLLM_AccountingEvent" -ADD COLUMN "compilation_state" TEXT NOT NULL DEFAULT 'pending', -ADD COLUMN "compilation_error" TEXT, -ADD COLUMN "compilation_retry_at" TIMESTAMP(3); -CREATE INDEX "LiteLLM_AccountingEvent_kind_compilation_state_recorded_at_idx" +ADD COLUMN IF NOT EXISTS "compilation_state" TEXT NOT NULL DEFAULT 'pending', +ADD COLUMN IF NOT EXISTS "compilation_error" TEXT, +ADD COLUMN IF NOT EXISTS "compilation_retry_at" TIMESTAMP(3); +CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_compilation_state_recorded_at_idx" ON "LiteLLM_AccountingEvent"("kind", "compilation_state", "recorded_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 45ea5cd20d0..c2f34e3b17e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2116,8 +2116,8 @@ model LiteLLM_AccountingEvent { compilation_retry_at DateTime? @@index([recorded_at]) - @@index([kind, source_event_id]) @@index([kind, compilation_state, recorded_at]) + @@index([kind, source_event_id]) } // One row per (event, target table, version): pending work for the projector and, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d50627cceb0..9f567a6c707 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2924,6 +2924,46 @@ def transcription_cost(model: str, custom_llm_provider: str | None, duration: fl return openai_cost_per_second(model=model, custom_llm_provider=custom_llm_provider, duration=duration) +def image_cost_model_candidates( + model: str, custom_llm_provider: str | None, quality: str | None, size: str | None +) -> tuple[str | None, ...]: + # Standardize size format to use "-x-" + size_str: str = size or "1024-x-1024" + size_str = size_str.replace("x", "-x-") if "x" in size_str and "-x-" not in size_str else size_str + + # Build model names for cost lookup + base_model_name = f"{size_str}/{model}" + model_name_without_custom_llm_provider: str | None = None + if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"): + model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") + base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" + model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name + provider_first_model_name_with_quality: Final = ( + f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" + if quality and custom_llm_provider + else None + ) + + # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family + model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" + + verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) + + model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" + model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider + + return ( + model_name_with_quality, + provider_first_model_name_with_quality, + base_model_name, + model_name_with_v2_quality, + model_with_quality_without_provider, + model_without_provider, + model, + model_name_without_custom_llm_provider, + ) + + def default_image_cost_calculator( model: str, custom_llm_provider: str | None = None, @@ -2950,44 +2990,9 @@ def default_image_cost_calculator( Raises: Exception: If model pricing not found in cost map """ - # Standardize size format to use "-x-" - size_str: str = size or "1024-x-1024" - size_str = size_str.replace("x", "-x-") if "x" in size_str and "-x-" not in size_str else size_str - - # Parse dimensions - height, width = map(int, size_str.split("-x-")) - - # Build model names for cost lookup - base_model_name = f"{size_str}/{model}" - model_name_without_custom_llm_provider: str | None = None - if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"): - model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") - base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" - model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name - provider_first_model_name_with_quality: Final = ( - f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" - if quality and custom_llm_provider - else None - ) - - # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family - model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" - - verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) - - model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" - model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider - - models_to_check: Final = ( - model_name_with_quality, - provider_first_model_name_with_quality, - base_model_name, - model_name_with_v2_quality, - model_with_quality_without_provider, - model_without_provider, - model, - model_name_without_custom_llm_provider, - ) + size_str: Final = (size or "1024x1024").replace("-x-", "x") + height, width = map(int, size_str.split("x")) + models_to_check: Final = image_cost_model_candidates(model, custom_llm_provider, quality, size) matched_model: Final = next( ( _model diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c87aaabc4b7..1dbcaf8b647 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2167,12 +2167,20 @@ class Logging(LiteLLMLoggingBaseClass): or ("embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat"), } ) - effective: Final = model_info_with_pricing_overrides(normalized, overrides) + effective: Final = model_info_with_pricing_overrides( + normalized, + { + name: value + for name, value in overrides.items() + if deployment_id is None or normalized.get(name) != value + }, + ) keys: Final = (model, f"{provider}/{model}", deployment_id or model, effective["key"]) self.request_pricing = RetainedPricing( models_json=_REQUEST_PRICING_MODELS.dump_json({name: effective for name in keys}).decode(), discounts_json=json.dumps(litellm.cost_discount_config), margins_json=json.dumps(litellm.cost_margin_config), + allow_live_models=True, ) def response_cost_calculator( @@ -2182,6 +2190,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name: str | None = None, router_model_id: str | None = None, ignore_stamped_cost: bool = False, + custom_pricing_override: bool | None = None, ) -> float | None: context: Final = ( retained_pricing(self.request_pricing) @@ -2190,7 +2199,7 @@ class Logging(LiteLLMLoggingBaseClass): ) with context, pinned_billing_time(self.billing_time_for_cost()): return self._calculate_response_cost( - result, cache_hit, litellm_model_name, router_model_id, ignore_stamped_cost + result, cache_hit, litellm_model_name, router_model_id, ignore_stamped_cost, custom_pricing_override ) def _calculate_response_cost( @@ -2200,6 +2209,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name: str | None = None, router_model_id: str | None = None, ignore_stamped_cost: bool = False, + custom_pricing_override: bool | None = None, ) -> float | None: """ Calculate response cost using result + logging object variables. @@ -2262,7 +2272,9 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - custom_pricing: Final = self.custom_pricing_for(priced_result) + custom_pricing: Final = ( + self.custom_pricing_for(priced_result) if custom_pricing_override is None else custom_pricing_override + ) prompt: Final = self._prompt_for_cost_calculation() diff --git a/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py index f9509cac41e..bb575da9bff 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py +++ b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py @@ -1,8 +1,9 @@ +from collections import ChainMap from collections.abc import Generator, Mapping from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass -from typing import Final +from typing import Final, cast # noqa: TID251 # live catalog has no typed boundary yet from pydantic import TypeAdapter @@ -18,6 +19,7 @@ class RetainedPricing: models_json: str discounts_json: str margins_json: str + allow_live_models: bool = False def models(self) -> Mapping[str, ModelInfo]: return _MODELS.validate_json(self.models_json) @@ -41,14 +43,23 @@ def retained_model_info(model: str, provider: str | None) -> ModelInfo | None: return None models: Final = context.models() result: Final = models.get(f"{provider}/{model}") or models.get(model) - if result is None: + if result is None and not context.allow_live_models: raise ValueError(f"Pricing snapshot has no retained model {model}") return result def retained_models() -> Mapping[str, ModelInfo] | None: + import litellm + context: Final = _CURRENT.get() - return None if context is None else context.models() + if context is None: + return None + if not context.allow_live_models: + return context.models() + catalog: Final = cast( # cast-ok: legacy catalog getter has an untyped dict signature + dict[str, ModelInfo], vars(litellm)["model_cost"] + ) + return ChainMap(dict(context.models()), catalog) def pricing_discounts() -> Mapping[str, float]: diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 0dbc3d8c86a..c321c21c06b 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -175,7 +175,7 @@ WHERE event_id = $1 AND kind = 'outbound_usage' _RETRY_COMPILATION: Final = """ UPDATE "LiteLLM_AccountingEvent" -SET compilation_error = 'calculation_retryable', compilation_retry_at = CURRENT_TIMESTAMP + INTERVAL '30 seconds' +SET compilation_error = $2, compilation_retry_at = CURRENT_TIMESTAMP + INTERVAL '30 seconds' WHERE event_id = $1 AND kind = 'outbound_usage' AND compilation_state = 'pending' """ @@ -373,6 +373,10 @@ class AccountingLedger: OutboundUsageEvent.model_validate_json(_json(_JSON.validate_json(row["payload"])["event"])) for row in rows ) + async def defer_compilation(self, event: OutboundUsageEvent) -> None: + async with self._transaction() as db: + await db.execute_raw(_RETRY_COMPILATION, event.event_id, "compilation_retryable") + async def compile_cost( self, event: OutboundUsageEvent, @@ -410,7 +414,7 @@ class AccountingLedger: except ValueError: return await _unpriced(db, validated.event_id, "invalid_pricing_inputs") except Exception: # noqa: BLE001 # retain failed pricing for bounded retries without starving newer events - await db.execute_raw(_RETRY_COMPILATION, validated.event_id) + await db.execute_raw(_RETRY_COMPILATION, validated.event_id, "calculation_retryable") return Unpriced(usage_event_id=validated.event_id, reason="calculation_retryable") if isinstance(priced, Unpriced): if priced.usage_event_id != validated.event_id: diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 45ea5cd20d0..c2f34e3b17e 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2116,8 +2116,8 @@ model LiteLLM_AccountingEvent { compilation_retry_at DateTime? @@index([recorded_at]) - @@index([kind, source_event_id]) @@index([kind, compilation_state, recorded_at]) + @@index([kind, source_event_id]) } // One row per (event, target table, version): pending work for the projector and, diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 52452a79106..2c3bd8754d7 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -28,6 +28,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo from litellm.proxy.db.accounting_ledger import AccountingLedger from litellm.proxy.spend_tracking.usage_meter import usage_delivery_targets from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse +from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CallTypes, CostBreakdown, @@ -71,6 +72,12 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: ) if not captured.charge_eligible or batch_summary: return Unpriced(usage_event_id=event.event_id, reason="not_charge_owner") + response_inputs: Final = _fields(captured.pricing.get("response_inputs")) + rerank_usage: Final = event.call_type in {CallTypes.rerank, CallTypes.arerank} and any( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for name, value in _fields(response_inputs.get("billed_units")).items() + if name in {"search_units", "total_tokens"} + ) logged: Final = _fields(captured.pricing.get("logging")) speech_characters: Final = logged.get("prompt_characters") speech_usage: Final = ( @@ -86,7 +93,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: realtime_usage: Final = event.call_type is CallTypes.arealtime and any( bool(item.get("usage")) for item in realtime_events ) - if captured.usage is None and not fixed_call and not realtime_usage and not speech_usage: + if captured.usage is None and not fixed_call and not realtime_usage and not speech_usage and not rerank_usage: return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") pricing: Final = captured.pricing mapped: Final = _fields(pricing.get("model_map_information")) @@ -146,7 +153,7 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: **_fields(_fields(snapshot).get("model_map_value")), "key": _fields(snapshot).get("model_map_key") or name, } - for name, snapshot in _fields(pricing.get("realtime_models")).items() + for name, snapshot in {**_fields(pricing.get("realtime_models")), **_fields(pricing.get("tool_models"))}.items() if _fields(snapshot).get("model_map_value") is not None } base_pricing: Final = _fields(pricing.get("base_model_information")) @@ -169,7 +176,6 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: _json({provider: discount} if provider and discount is not None else {}), _json({"global": margin} if margin is not None else {}), ) - response_inputs: Final = _fields(pricing.get("response_inputs")) tool_response: Final = _fields(pricing.get("tool_response")) replay_usage: Final = Usage.model_validate( { @@ -215,6 +221,10 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: } ) if event.call_type in {CallTypes.arealtime, CallTypes.aresponses_websocket} + else RerankResponse.model_validate( + {"id": event.event_id, "results": [], "meta": {"billed_units": response_inputs.get("billed_units")}} + ) + if event.call_type in {CallTypes.rerank, CallTypes.arerank} else ResponsesAPIResponse.model_validate( { "id": event.event_id, @@ -222,6 +232,9 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: "model": model, "output": tool_response.get("output", []), "usage": tool_response.get("usage"), + "service_tier": response_inputs.get("service_tier"), + "tools": tool_response.get("tools"), + "tool_usage": tool_response.get("tool_usage"), } ) if tool_response.get("responses_api") @@ -298,7 +311,13 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: amount: Final = ( sum(batch_costs) if batch_costs is not None - else logger.response_cost_calculator(response, ignore_stamped_cost=True) + else logger.response_cost_calculator( + response, + ignore_stamped_cost=True, + custom_pricing_override=TypeAdapter[bool | None](bool | None).validate_python( + logged.get("custom_pricing") + ), + ) ) if batch_costs is not None: logger.cost_breakdown = CostBreakdown( @@ -396,6 +415,12 @@ class LoggingCostCompiler: await self.compile(event) except Exception: # noqa: BLE001 # retain pending work for retry without starving other events verbose_proxy_logger.exception("Cost compilation failed for %s", event.event_id) + try: + await self._ledger.defer_compilation(event) + except Exception: # noqa: BLE001 # database outages must not prevent attempts on later events + verbose_proxy_logger.exception( + "Cost compilation retry could not be scheduled for %s", event.event_id + ) _SPEND_TARGETS: Final = ( diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 4fede891018..edb7015f9a4 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, replace from datetime import datetime, timezone from decimal import Decimal from hashlib import sha256 -from itertools import chain, groupby, islice, repeat +from itertools import chain, groupby, islice, product, repeat from typing import TYPE_CHECKING, Final from pydantic import BaseModel, JsonValue, TypeAdapter @@ -828,7 +828,18 @@ async def capture_proxy_response(logging_obj: object, response: object, *, backg await compiler.compile(write.event) except Exception as error: # noqa: BLE001 # the configured response policy owns every capture failure verbose_proxy_logger.exception("Usage capture failed before the response boundary") - if sync_ack and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed": + accepted: Final = logging_obj.model_call_details.get("_accounting_capture") + committed: Final = ( + isinstance(accepted, CapturedUsage) + and accepted.receipt.done() + and not accepted.receipt.cancelled() + and accepted.receipt.exception() is None + ) + if ( + sync_ack + and not committed + and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed" + ): acknowledgement: Final = UsageAcknowledgementError(error) raise HTTPException(503, str(acknowledgement)) from acknowledgement @@ -911,7 +922,7 @@ class PricingCapture: def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[str, object]) -> PricingCapture: from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_pricing - if logging_obj.request_pricing is not None: + if logging_obj.request_pricing is not None and not logging_obj.spilled_over_ptu_for(normalized): with retained_pricing(logging_obj.request_pricing): return _capture_effective_pricing(logging_obj, normalized, details) return _capture_effective_pricing(logging_obj, normalized, details) @@ -934,7 +945,7 @@ def _capture_effective_pricing( if prepared is not None else StandardLoggingPayloadSetup.get_model_cost_information( base_model=logging_obj.get_deployment_model_for_cost(), - custom_pricing=bool(params.get("custom_pricing")), + custom_pricing=logging_obj.custom_pricing_for(normalized), custom_llm_provider=_text(details.get("custom_llm_provider")), init_response_obj=normalized, api_base=_text(params.get("api_base")), @@ -996,6 +1007,16 @@ def _capture_effective_pricing( ) for model in realtime_models }, + "tool_models": { + model: StandardLoggingPayloadSetup.get_model_cost_information( + base_model=model, + custom_pricing=False, + custom_llm_provider=provider, + init_response_obj={"model": model}, + ) + for model in _image_tool_pricing_models(normalized, provider) + if model in _fields(vars(litellm)["model_cost"]) + }, "batch_models": { model: StandardLoggingPayloadSetup.get_model_cost_information( base_model=model, @@ -1026,16 +1047,64 @@ def _tool_cost_inputs(response: object) -> Mapping[str, object]: return { **flags, "responses_api": True, + "tools": tuple( + {key: value for key, value in _fields(tool).items() if key in {"type", "model"}} + for tool in TypeAdapter(tuple[object, ...]).validate_python(raw.get("tools") or ()) + if _fields(tool).get("type") == "image_generation" + ), + "tool_usage": _response_fields(raw.get("tool_usage")) or None, "usage": _response_fields(raw.get("usage")), "output": tuple( - {key: value for key, value in _fields(item).items() if key in {"type", "id", "status", "usage"}} + { + key: value + for key, value in _fields(item).items() + if key in {"type", "id", "status", "usage", "quality", "size"} + } for item in TypeAdapter(tuple[object, ...]).validate_python(raw.get("output") or ()) ), } -def _realtime_cost_fields(value: object) -> Mapping[str, JsonValue]: - fields: Final = _fields(value) +def _image_tool_pricing_models(response: object, provider: str | None) -> tuple[str, ...]: + from litellm.cost_calculator import image_cost_model_candidates + + raw: Final = _tool_cost_inputs(response) + models: Final = tuple( + model + for tool in TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(raw.get("tools") or ()) + if isinstance(model := tool.get("model"), str) + ) + items: Final = tuple( + item + for item in TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(raw.get("output") or ()) + if item.get("type") == "image_generation_call" and item.get("status") == "completed" + ) + candidates: Final = chain.from_iterable( + image_cost_model_candidates( + model, + provider, + _text(item.get("quality")) if item.get("quality") != "auto" else None, + _text(item.get("size")) if item.get("size") != "auto" else None, + ) + for model, item in product(models, items) + ) + return tuple(dict.fromkeys(model for model in candidates if model is not None)) + + +_REALTIME_CHILDREN: Final = ( + "response", + "event", + "session", + "audio", + "input", + "transcription", + "input_audio_transcription", +) + + +def _realtime_cost_node( + fields: Mapping[str, object], +) -> dict[str, JsonValue]: # mutable-ok: tree builder attaches sanitized children return { **{ key: _JSON_OBJECT.validate_python(fields[key]) for key in ("usage",) if isinstance(fields.get(key), Mapping) @@ -1045,14 +1114,26 @@ def _realtime_cost_fields(value: object) -> Mapping[str, JsonValue]: for key, value in fields.items() if key in {"type", "id", "model", "service_tier"} and isinstance(value, str) }, - **{ - key: dict(_realtime_cost_fields(fields[key])) - for key in ("response", "event", "session", "audio", "input", "transcription", "input_audio_transcription") - if isinstance(fields.get(key), Mapping) - }, } +def _realtime_cost_fields(value: object) -> Mapping[str, JsonValue]: + root: Final = _realtime_cost_node(_fields(value)) + pending: Final[list[tuple[Mapping[str, object], dict[str, JsonValue]]]] = [ # mutable-ok: traversal stack + (_fields(value), root) + ] + while pending: + source, target = pending.pop() + for key in _REALTIME_CHILDREN: + if not isinstance(source.get(key), Mapping): + continue + fields: Final = _fields(source[key]) # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + child: Final = _realtime_cost_node(fields) # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + target[key] = child + pending.append((fields, child)) + return root + + def _realtime_cost_events(response: object) -> tuple[Mapping[str, JsonValue], ...]: results: Final = _response_fields(response).get("results") return ( @@ -1063,12 +1144,15 @@ def _realtime_cost_events(response: object) -> tuple[Mapping[str, JsonValue], .. def _realtime_pricing_models(value: Mapping[str, JsonValue]) -> Iterator[str]: - model: Final = value.get("model") - if isinstance(model, str): - yield model - for key in ("response", "event", "session", "audio", "input", "transcription", "input_audio_transcription"): - if isinstance(nested := value.get(key), dict): - yield from _realtime_pricing_models(nested) + pending: Final[list[Mapping[str, JsonValue]]] = [ # mutable-ok: iterative traversal avoids recursion + value + ] + while pending: + fields: Final = pending.pop() # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + model: Final = fields.get("model") # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + if isinstance(model, str): + yield model + pending.extend(nested for key in _REALTIME_CHILDREN if isinstance(nested := fields.get(key), dict)) def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f4763acaeda..a5e2bb1eb68 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -488,6 +488,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "extraction", "ocr", "realtime", + "rerank", ] ] supported_endpoints: list[str] | None diff --git a/schema.prisma b/schema.prisma index 45ea5cd20d0..c2f34e3b17e 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2116,8 +2116,8 @@ model LiteLLM_AccountingEvent { compilation_retry_at DateTime? @@index([recorded_at]) - @@index([kind, source_event_id]) @@index([kind, compilation_state, recorded_at]) + @@index([kind, source_event_id]) } // One row per (event, target table, version): pending work for the projector and, diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 0171d3683ca..0dc27b09991 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -995,7 +995,10 @@ async def test_proxy_capture_and_background_compiler_converge_on_final_header_ch @pytest.mark.asyncio -async def test_failing_compilation_page_does_not_starve_later_usage(writer: _Writer, ledger_database: str) -> None: +@pytest.mark.parametrize("conflicting_result", (False, True)) +async def test_failing_compilation_page_does_not_starve_later_usage( + writer: _Writer, ledger_database: str, conflicting_result: bool +) -> None: _, store = writer broken: Final = _usage("broken-price") healthy: Final = _usage("healthy-price") @@ -1004,12 +1007,15 @@ async def test_failing_compilation_page_does_not_starve_later_usage(writer: _Wri await _insert(store, broken) await _insert(store, healthy) - def price(envelope: str) -> PricingSnapshot: + def price(envelope: str) -> PricingSnapshot | Unpriced: if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == broken.event_id: + if conflicting_result: + return Unpriced(usage_event_id="different-usage", reason="usage_unknown") raise RuntimeError("temporary calculator failure") return _cost().snapshot await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) + assert await store.pending_compilation(limit=1) == (healthy,) _rows( ledger_database, 'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = CURRENT_TIMESTAMP WHERE event_id = %s', diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 1b89f472248..0e9c8f48609 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1348,3 +1348,230 @@ def test_character_pricing_does_not_invent_zero_for_missing_or_invalid_quantity( assert price_usage(json.dumps({**original, "event": event.model_dump(mode="json")})) == Unpriced( usage_event_id=event.event_id, reason="usage_unknown" ) + + +@pytest.mark.parametrize("measured_tokens", (False, True)) +def test_rerank_replay_preserves_billable_search_units(measured_tokens: bool) -> None: + from pydantic import TypeAdapter + + original: Final = _retained_envelope( + rates={"mode": "rerank", "input_cost_per_query": 0.03}, + response_inputs={"billed_units": {"search_units": 3, "total_tokens": 14}}, + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={ + "call_type": CallTypes.arerank, + "dimensions": ( + Dimension(name=DimensionName.MODEL, value="cohere/retained-reranker"), + Dimension(name=DimensionName.PROVIDER, value="cohere"), + ), + } + ) + pricing: Final = TypeAdapter(dict[str, object]).validate_python(original["pricing"]) + result: Final = price_usage( + json.dumps( + { + **original, + "usage": original["usage"] if measured_tokens else None, + "event": event.model_dump(mode="json"), + "pricing": pricing, + } + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(3 * 0.03) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("served_tier", ("default", "priority")) +async def test_responses_replay_uses_served_tier(served_tier: str) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger("aresponses") + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_token_priority": 0.03, + "output_cost_per_token_priority": 0.04, + }, + ) + logger.update_environment_variables( + model="capture-model", + custom_llm_provider="openai", + litellm_params={}, + optional_params={"service_tier": "priority" if served_tier == "default" else "default"}, + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "tier", + "created_at": 1, + "model": "capture-model", + "output": [], + "service_tier": served_tier, + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + } + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost == pytest.approx(0.18 if served_tier == "default" else 0.46) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("token_pricing", "automatic_size"), ((False, False), (False, True), (True, False))) +@pytest.mark.parametrize("custom_primary", (False, True)) +async def test_responses_image_tool_replays_retained_prices_without_image_content( + monkeypatch: pytest.MonkeyPatch, token_pricing: bool, custom_primary: bool, automatic_size: bool +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setitem( + litellm.model_cost, + "capture-model", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + ) + monkeypatch.setitem( + litellm.model_cost, + "retained-image", + { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 0.03, + "output_cost_per_image_token": 0.04, + }, + ) + image_price_key: Final = f"{'high' if automatic_size else 'low'}/1024-x-1024/retained-image" + monkeypatch.setitem( + litellm.model_cost, + image_price_key, + {"litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image": 0.25}, + ) + logger: Final = _logger("aresponses") + if custom_primary: + logger.set_request_pricing( + "capture-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "image-tool", + "created_at": 1, + "model": "capture-model", + "tools": [{"type": "image_generation", "model": "retained-image"}], + "output": [ + { + "id": "image", + "type": "image_generation_call", + "status": "completed", + "quality": "auto" if automatic_size else "low", + "size": "auto" if automatic_size else "1024x1024", + "result": "private-image-content", + } + ], + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + "tool_usage": { + "image_gen": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"text_tokens": 10, "image_tokens": 0}, + "output_tokens_details": {"text_tokens": 0, "image_tokens": 5}, + } + } + if token_pricing + else None, + } + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost == pytest.approx(0.18 + (0.5 if token_pricing else 0.25)) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private-image-content" not in captured + monkeypatch.delitem(litellm.model_cost, "retained-image") + monkeypatch.delitem(litellm.model_cost, image_price_key) + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("spilled_over", (False, True)) +async def test_capture_replays_effective_ptu_spillover_prices( + monkeypatch: pytest.MonkeyPatch, spilled_over: bool +) -> None: + import litellm + from litellm.litellm_core_utils.hidden_params import set_hidden_param + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "true") + monkeypatch.setitem( + litellm.model_cost, + "azure/capture-model", + {"litellm_provider": "azure", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + ) + logger: Final = _logger() + logger.update_environment_variables( + model="capture-model", + custom_llm_provider="azure", + optional_params={}, + litellm_params={ + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "metadata": { + "model_info": { + "team_id": "team", + "ptu_count": 10, + "cost_per_ptu_per_hour": 1.0, + "ptu_effective_from": "2026-01-01", + } + }, + }, + ) + logger.set_request_pricing("capture-model", "azure", {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}) + response: Final = ModelResponse( + id="spill", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14) + ) + set_hidden_param( + response, "additional_headers", {"llm_provider-x-ms-is-spilled-over": "true"} if spilled_over else {} + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost == pytest.approx(0.18 if spilled_over else 0) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index c83d11250c9..a1327e0a177 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1725,8 +1725,9 @@ class DelayedCostCompiler: @pytest.mark.asyncio -async def test_sync_ack_deadline_bounds_compilation_after_usage_commit(monkeypatch: pytest.MonkeyPatch) -> None: - from fastapi import HTTPException +async def test_sync_ack_preserves_committed_usage_ack_when_compilation_times_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: from litellm.proxy import proxy_server from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response @@ -1742,11 +1743,9 @@ async def test_sync_ack_deadline_bounds_compilation_after_usage_commit(monkeypat proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.1} ) try: - with pytest.raises(HTTPException) as error: - await asyncio.wait_for( - capture_proxy_response(_logger(), ModelResponse(id="provider", usage=Usage(prompt_tokens=3))), 2 - ) - assert error.value.status_code == 503 + await asyncio.wait_for( + capture_proxy_response(_logger(), ModelResponse(id="provider", usage=Usage(prompt_tokens=3))), 2 + ) assert compiler.entered.is_set() assert compiler.cancelled.is_set() assert database.committed == ("caller:inbound", "provider") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 5cf872f4ea7..add903ad561 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2333,7 +2333,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_keeps_custom_prices_scoped_to_the_request() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2376,8 +2376,8 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018) From f1b655988734d784733cc8797d1843fe8f2d212d Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 14:15:57 -0700 Subject: [PATCH 13/20] fix(accounting): retain explicit tier prices during request overrides --- .../migration.sql | 2 + litellm/litellm_core_utils/litellm_logging.py | 4 +- .../test_register_model_custom_pricing.py | 41 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql index 5bdedb25c25..50c9c269ead 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql @@ -1,3 +1,5 @@ +SET LOCAL lock_timeout = '10s'; + ALTER TABLE "LiteLLM_AccountingEvent" ADD COLUMN IF NOT EXISTS "compilation_state" TEXT NOT NULL DEFAULT 'pending', ADD COLUMN IF NOT EXISTS "compilation_error" TEXT, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 1dbcaf8b647..30061050f66 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2172,7 +2172,9 @@ class Logging(LiteLLMLoggingBaseClass): { name: value for name, value in overrides.items() - if deployment_id is None or normalized.get(name) != value + if deployment_id is None + or name not in {"input_cost_per_token", "output_cost_per_token"} + or normalized.get(name) != value }, ) keys: Final = (model, f"{provider}/{model}", deployment_id or model, effective["key"]) diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 9ee21032343..480c2c0f441 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1766,3 +1766,44 @@ def test_router_custom_token_prices_accept_missing_deployment_mode(explicit_null assert _snapshot_model_cost_entries(keys) == registered finally: _restore_model_cost_entries(original) + + +def test_changed_request_flat_rate_preserves_explicit_deployment_tier_prices() -> None: + from litellm.types.utils import ModelResponse, Usage + + deployment_id: Final = "request-price-variants" + original: Final = _snapshot_model_cost_entries((deployment_id,)) + variants: Final = {"input_cost_per_token_priority": 0.03, "output_cost_per_token_priority": 0.04} + try: + litellm.register_model( + { + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + **variants, + } + }, + persist_across_reloads=False, + ) + logger: Final = _request_logger("openai/gpt-4o-mini", "completion") + logger.update_environment_variables( + model="gpt-4o-mini", custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + logger.set_request_pricing( + "gpt-4o-mini", + "openai", + {"input_cost_per_token": 0.05, "output_cost_per_token": 0.02, **variants}, + deployment_id, + ) + response: Final = ModelResponse( + model="gpt-4o-mini", + service_tier="priority", + usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14), + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx( + 10 * 0.03 + 4 * 0.04 + ) + finally: + _restore_model_cost_entries(original) From 8b73a8afe789ebd51448bc35525e24fd53b52eca Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:32:09 -0700 Subject: [PATCH 14/20] fix(accounting): preserve image and live adjustment pricing parity --- .../llm_cost_calc/pricing_context.py | 12 ++- litellm/proxy/spend_tracking/cost_compiler.py | 15 ++- litellm/proxy/spend_tracking/usage_capture.py | 10 +- .../test_sagemaker_completion_handler.py | 15 +-- .../spend_tracking/test_cost_compiler.py | 98 +++++++++++++++++++ .../test_register_model_custom_pricing.py | 63 ++++++++++++ 6 files changed, 203 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py index bb575da9bff..9cc9255cfeb 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py +++ b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py @@ -66,11 +66,19 @@ def pricing_discounts() -> Mapping[str, float]: import litellm context: Final = _CURRENT.get() - return litellm.cost_discount_config if context is None else _DISCOUNTS.validate_json(context.discounts_json) + return ( + litellm.cost_discount_config + if context is None or context.allow_live_models + else _DISCOUNTS.validate_json(context.discounts_json) + ) def pricing_margins() -> Mapping[str, float | Mapping[str, float]]: import litellm context: Final = _CURRENT.get() - return litellm.cost_margin_config if context is None else _MARGINS.validate_json(context.margins_json) + return ( + litellm.cost_margin_config + if context is None or context.allow_live_models + else _MARGINS.validate_json(context.margins_json) + ) diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py index 2c3bd8754d7..1afa83af83f 100644 --- a/litellm/proxy/spend_tracking/cost_compiler.py +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -33,6 +33,7 @@ from litellm.types.utils import ( CallTypes, CostBreakdown, CostPerToken, + ImageResponse, LiteLLMRealtimeStreamLoggingObject, ModelInfo, ModelResponse, @@ -93,7 +94,15 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: realtime_usage: Final = event.call_type is CallTypes.arealtime and any( bool(item.get("usage")) for item in realtime_events ) - if captured.usage is None and not fixed_call and not realtime_usage and not speech_usage and not rerank_usage: + image_inputs: Final = _fields(captured.pricing.get("image_response")) + if ( + captured.usage is None + and not fixed_call + and not realtime_usage + and not speech_usage + and not rerank_usage + and not image_inputs + ): return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") pricing: Final = captured.pricing mapped: Final = _fields(pricing.get("model_map_information")) @@ -238,6 +247,10 @@ def price_usage(envelope: str) -> PricingSnapshot | Unpriced: } ) if tool_response.get("responses_api") + else ImageResponse.model_validate( + {"created": int(event.started_at.timestamp()), "model": model, **image_inputs} + ) + if image_inputs else token_response ) logger: Final = Logging( diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index edb7015f9a4..001d0713e56 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -31,7 +31,7 @@ from litellm.proxy.spend_tracking.usage_meter import ( UsageCaptureSettings, usage_delivery_targets, ) -from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams +from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -1159,6 +1159,14 @@ def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: from litellm.litellm_core_utils.litellm_logging import Logging return { + "image_response": { + "data": tuple(dict[str, JsonValue]() for _ in response.data or ()), + "usage": _response_fields(response.usage) if response.usage is not None else None, + "size": response.size, + "quality": response.quality, + } + if isinstance(response, ImageResponse) + else None, "tool_response": _tool_cost_inputs(response), "realtime_events": _realtime_cost_events(response), "request": _selected_pricing(kwargs), diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 5e40c2ebb60..496eb5fe25b 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -298,8 +298,13 @@ def test_installed_botocore_signs_the_completion_request(): from litellm.llms.sagemaker.completion.handler import SagemakerLLM request = SagemakerLLM()._prepare_request( - credentials=Credentials("test-key", "test-secret"), model="test-endpoint", data={"inputs": "ping"}, - messages=[], litellm_params={}, optional_params={}, aws_region_name="us-west-2", + credentials=Credentials("test-key", "test-secret"), + model="test-endpoint", + data={"inputs": "ping"}, + messages=[], + litellm_params={}, + optional_params={}, + aws_region_name="us-west-2", ) assert request.body == b'{"inputs": "ping"}' assert "/us-west-2/sagemaker/aws4_request" in request.headers["Authorization"] @@ -447,9 +452,7 @@ def test_completion_sagemaker_prompt_template_non_stream( }, ) ) - url: Final = ( - "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/deepseek_coder_6.7_instruct/invocations" - ) + url: Final = "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/deepseek_coder_6.7_instruct/invocations" route: Final = respx_mock.post(url).mock( return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) ) @@ -549,7 +552,7 @@ async def test_completion_sagemaker( assert route.called assert response.choices[0].message.content == "SageMaker reply" - assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate) @pytest.mark.asyncio diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 0e9c8f48609..083ac2e0355 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1575,3 +1575,101 @@ async def test_capture_replays_effective_ptu_spillover_prices( assert float(result.total_cost) == pytest.approx(source_cost) finally: await meter.stop(1) + + +@pytest.mark.asyncio +async def test_live_adjustment_reload_is_captured_and_replay_remains_frozen(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {}) + monkeypatch.setattr(litellm, "cost_margin_config", {}) + logger: Final = _capture_logger() + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.02}}) + response: Final = ModelResponse(model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + amount: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert amount == pytest.approx(0.18 * 0.9 + 0.02) + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.9}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 99.0}}) + replay: Final = price_usage(database.rows[-1].envelope) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(amount) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token_pricing", (True, False)) +async def test_image_capture_replays_the_existing_image_calculator( + monkeypatch: pytest.MonkeyPatch, token_pricing: bool +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ImageResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.2}}) + model: Final = "gpt-image-capture" if token_pricing else "dall-e-capture" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "image_generation", + **( + {"input_cost_per_token": 0.01, "output_cost_per_image_token": 0.02} + if token_pricing + else {"output_cost_per_image": 0.25} + ), + }, + ) + logger: Final = _logger("aimage_generation") + logger.model = model + logger.update_environment_variables( + model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + response: Final = ImageResponse.model_validate( + { + "created": 1, + "model": model, + "data": [{"b64_json": "private-image-content"}] * 2, + "size": "1024x1024", + "quality": "standard", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"text_tokens": 10, "image_tokens": 0}, + } + if token_pricing + else None, + } + ) + source: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source == pytest.approx(0.2 if token_pricing else 0.5) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + envelope: Final = database.rows[-1].envelope + assert "private-image-content" not in envelope + monkeypatch.delitem(litellm.model_cost, model) + replay: Final = price_usage(envelope) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(source) + finally: + await meter.stop(1) diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 480c2c0f441..5d77492ee58 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1807,3 +1807,66 @@ def test_changed_request_flat_rate_preserves_explicit_deployment_tier_prices() - ) finally: _restore_model_cost_entries(original) + + +@pytest.mark.parametrize("call_type", ("completion", "embedding")) +def test_sdk_followup_prices_change_only_after_explicit_registration(call_type: str) -> None: + model: Final = f"openai/followup-{call_type}-prices" + original: Final = _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) + models_before: Final = frozenset(litellm.open_ai_chat_completion_models) + logger: Final = _request_logger(model, call_type) + try: + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat" if call_type == "completion" else "embedding", + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + } + }, + persist_across_reloads=False, + ) + + def request(prices: dict[str, float], *, logging_obj: Logging | None = None) -> object: + if call_type == "completion": + return litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="hello back", + litellm_logging_obj=logging_obj, + **prices, + ) + return litellm.embedding( + model=model, + input=["hello"], + api_key="fake-key", + mock_response=[0.1, 0.2], + litellm_logging_obj=logging_obj, + **prices, + ) + + first: Final = request({"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, logging_obj=logger) + second: Final = request({}) + assert logger.response_cost_calculator(first, ignore_stamped_cost=True) == pytest.approx( + 0.05 if call_type == "completion" else 0.01 + ) + assert litellm.completion_cost(completion_response=second, model=model, call_type=call_type) == pytest.approx( + 0.005 if call_type == "completion" else 0.001 + ) + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + litellm.register_model( + {model: {"input_cost_per_token": 0.0003, "output_cost_per_token": 0.0006}}, + persist_across_reloads=False, + ) + third: Final = request({}) + assert litellm.completion_cost(completion_response=third, model=model, call_type=call_type) == pytest.approx( + 0.015 if call_type == "completion" else 0.003 + ) + assert logger.response_cost_calculator(first, ignore_stamped_cost=True) == pytest.approx( + 0.05 if call_type == "completion" else 0.01 + ) + finally: + _restore_model_cost_entries(original) + litellm.open_ai_chat_completion_models.intersection_update(models_before) From 0010c2d4437d82290387f466f6eec3b874b651ce Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:41:08 -0700 Subject: [PATCH 15/20] fix(accounting): retain billable image dimensions for replay --- litellm/proxy/spend_tracking/usage_capture.py | 13 +++++- .../spend_tracking/test_cost_compiler.py | 45 ++++++++++++------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 001d0713e56..0c67d09ccb4 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -1155,12 +1155,23 @@ def _realtime_pricing_models(value: Mapping[str, JsonValue]) -> Iterator[str]: pending.extend(nested for key in _REALTIME_CHILDREN if isinstance(nested := fields.get(key), dict)) +def _image_cost_dimensions(image: object) -> Mapping[str, object]: + fields: Final = _fields(_response_fields(image).get("provider_specific_fields")) + return { + "provider_specific_fields": { + name: value + for name, value in fields.items() + if name in {"width", "height"} and isinstance(value, int) and not isinstance(value, bool) + } + } + + def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]: from litellm.litellm_core_utils.litellm_logging import Logging return { "image_response": { - "data": tuple(dict[str, JsonValue]() for _ in response.data or ()), + "data": tuple(_image_cost_dimensions(image) for image in response.data or ()), "usage": _response_fields(response.usage) if response.usage is not None else None, "size": response.size, "quality": response.quality, diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 083ac2e0355..05065d6cb67 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1609,9 +1609,12 @@ async def test_live_adjustment_reload_is_captured_and_replay_remains_frozen(monk @pytest.mark.asyncio -@pytest.mark.parametrize("token_pricing", (True, False)) +@pytest.mark.parametrize( + ("pricing", "width", "expected"), + (("tokens", 2048, 0.2), ("images", 2048, 0.5), ("pixels", 2048, 0.4), ("pixels", 2049, 0.6)), +) async def test_image_capture_replays_the_existing_image_calculator( - monkeypatch: pytest.MonkeyPatch, token_pricing: bool + monkeypatch: pytest.MonkeyPatch, pricing: str, width: int, expected: float ) -> None: import litellm from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage @@ -1622,30 +1625,42 @@ async def test_image_capture_replays_the_existing_image_calculator( monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.2}}) - model: Final = "gpt-image-capture" if token_pricing else "dall-e-capture" + model: Final = {"tokens": "gpt-image-capture", "images": "dall-e-capture", "pixels": "fal-ai/flux/dev"}[pricing] + provider: Final = "fal_ai" if pricing == "pixels" else "openai" + catalog_key: Final = f"{provider}/{model}" if pricing == "pixels" else model monkeypatch.setitem( litellm.model_cost, - model, + catalog_key, { - "litellm_provider": "openai", + "litellm_provider": provider, "mode": "image_generation", - **( - {"input_cost_per_token": 0.01, "output_cost_per_image_token": 0.02} - if token_pricing - else {"output_cost_per_image": 0.25} - ), + **{ + "tokens": {"input_cost_per_token": 0.01, "output_cost_per_image_token": 0.02}, + "images": {"output_cost_per_image": 0.25}, + "pixels": {"output_cost_per_image": 0.1, "output_cost_per_pixel": 0.1 / (1024 * 1024)}, + }[pricing], }, ) logger: Final = _logger("aimage_generation") logger.model = model logger.update_environment_variables( - model=model, custom_llm_provider="openai", litellm_params={}, optional_params={} + model=model, custom_llm_provider=provider, litellm_params={}, optional_params={} ) response: Final = ImageResponse.model_validate( { "created": 1, "model": model, - "data": [{"b64_json": "private-image-content"}] * 2, + "data": [ + { + "b64_json": "private-image-content", + "provider_specific_fields": { + "width": width, + "height": 1024, + "secret": "private-image-content", + }, + } + ] + * 2, "size": "1024x1024", "quality": "standard", "usage": { @@ -1654,12 +1669,12 @@ async def test_image_capture_replays_the_existing_image_calculator( "total_tokens": 15, "input_tokens_details": {"text_tokens": 10, "image_tokens": 0}, } - if token_pricing + if pricing == "tokens" else None, } ) source: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) - assert source == pytest.approx(0.2 if token_pricing else 0.5) + assert source == pytest.approx(expected) database: Final = StoredUsage() meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) try: @@ -1667,7 +1682,7 @@ async def test_image_capture_replays_the_existing_image_calculator( await meter.close() envelope: Final = database.rows[-1].envelope assert "private-image-content" not in envelope - monkeypatch.delitem(litellm.model_cost, model) + monkeypatch.delitem(litellm.model_cost, catalog_key) replay: Final = price_usage(envelope) assert isinstance(replay, PricingSnapshot) assert float(replay.total_cost) == pytest.approx(source) From c1bffc852cac6771545eb28aff52037ff0b5b213 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 15:50:26 -0700 Subject: [PATCH 16/20] fix(accounting): retain image size and quality pricing tiers --- .../litellm_core_utils/llm_cost_calc/utils.py | 20 ++++++++--- litellm/proxy/spend_tracking/usage_capture.py | 24 +++++++++++-- .../spend_tracking/test_cost_compiler.py | 36 ++++++++++++++++--- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 50c26f4d4ca..36a76811546 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -149,6 +149,19 @@ def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None +def image_response_size_and_quality( + response: ImageResponse, + optional_params: Mapping[str, object] | None = None, + *, + size: str | None = None, + quality: str | None = None, +) -> tuple[str, str]: + return ( + size or response.size or _requested_image_size(optional_params) or "1024-x-1024", + quality or response.quality or _requested_image_param(optional_params, "quality") or "standard", + ) + + def get_web_search_requests(server_tool_use: object) -> int | None: """ Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value @@ -2187,11 +2200,8 @@ class CostCalculatorUtils: cost_calculator as vertex_ai_image_cost_calculator, ) - resolved_size: Final = ( - size or completion_response.size or _requested_image_size(optional_params) or "1024-x-1024" - ) - resolved_quality: Final = ( - quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard" + resolved_size, resolved_quality = image_response_size_and_quality( + completion_response, optional_params, size=size, quality=quality ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 0c67d09ccb4..22ea090629a 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -1014,7 +1014,12 @@ def _capture_effective_pricing( custom_llm_provider=provider, init_response_obj={"model": model}, ) - for model in _image_tool_pricing_models(normalized, provider) + for model in _image_pricing_models( + normalized, + provider, + logging_obj.get_deployment_model_for_cost(), + _fields(details.get("optional_params")), + ) if model in _fields(vars(litellm)["model_cost"]) }, "batch_models": { @@ -1065,9 +1070,24 @@ def _tool_cost_inputs(response: object) -> Mapping[str, object]: } -def _image_tool_pricing_models(response: object, provider: str | None) -> tuple[str, ...]: +def _image_pricing_models( + response: object, provider: str | None, deployment_model: str | None, optional_params: Mapping[str, object] +) -> tuple[str, ...]: from litellm.cost_calculator import image_cost_model_candidates + from litellm.litellm_core_utils.llm_cost_calc.utils import image_response_size_and_quality + if isinstance(response, ImageResponse) and deployment_model is not None: + size, quality = image_response_size_and_quality(response, optional_params) + return tuple( + key + for key in image_cost_model_candidates( + deployment_model, + provider, + quality, + size, + ) + if key is not None + ) raw: Final = _tool_cost_inputs(response) models: Final = tuple( model diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 05065d6cb67..59be2d64d3f 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1611,7 +1611,14 @@ async def test_live_adjustment_reload_is_captured_and_replay_remains_frozen(monk @pytest.mark.asyncio @pytest.mark.parametrize( ("pricing", "width", "expected"), - (("tokens", 2048, 0.2), ("images", 2048, 0.5), ("pixels", 2048, 0.4), ("pixels", 2049, 0.6)), + ( + ("tokens", 2048, 0.2), + ("images", 2048, 0.5), + ("pixels", 2048, 0.4), + ("pixels", 2049, 0.6), + ("tiered", 2048, 1.5), + ("tiered-request", 2048, 1.5), + ), ) async def test_image_capture_replays_the_existing_image_calculator( monkeypatch: pytest.MonkeyPatch, pricing: str, width: int, expected: float @@ -1625,7 +1632,13 @@ async def test_image_capture_replays_the_existing_image_calculator( monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.2}}) - model: Final = {"tokens": "gpt-image-capture", "images": "dall-e-capture", "pixels": "fal-ai/flux/dev"}[pricing] + model: Final = { + "tokens": "gpt-image-capture", + "images": "dall-e-capture", + "pixels": "fal-ai/flux/dev", + "tiered": "dall-e-capture", + "tiered-request": "dall-e-capture", + }[pricing] provider: Final = "fal_ai" if pricing == "pixels" else "openai" catalog_key: Final = f"{provider}/{model}" if pricing == "pixels" else model monkeypatch.setitem( @@ -1637,14 +1650,26 @@ async def test_image_capture_replays_the_existing_image_calculator( **{ "tokens": {"input_cost_per_token": 0.01, "output_cost_per_image_token": 0.02}, "images": {"output_cost_per_image": 0.25}, + "tiered": {"output_cost_per_image": 0.25}, + "tiered-request": {"output_cost_per_image": 0.25}, "pixels": {"output_cost_per_image": 0.1, "output_cost_per_pixel": 0.1 / (1024 * 1024)}, }[pricing], }, ) + tier_key: Final = f"hd/1024-x-1024/{model}" + if pricing.startswith("tiered"): + monkeypatch.setitem( + litellm.model_cost, + tier_key, + {"litellm_provider": provider, "mode": "image_generation", "output_cost_per_image": 0.75}, + ) logger: Final = _logger("aimage_generation") logger.model = model logger.update_environment_variables( - model=model, custom_llm_provider=provider, litellm_params={}, optional_params={} + model=model, + custom_llm_provider=provider, + litellm_params={}, + optional_params={"size": "1024*1024", "quality": "hd"} if pricing == "tiered-request" else {}, ) response: Final = ImageResponse.model_validate( { @@ -1661,8 +1686,8 @@ async def test_image_capture_replays_the_existing_image_calculator( } ] * 2, - "size": "1024x1024", - "quality": "standard", + "size": None if pricing == "tiered-request" else "1024x1024", + "quality": None if pricing == "tiered-request" else "hd" if pricing == "tiered" else "standard", "usage": { "input_tokens": 10, "output_tokens": 5, @@ -1683,6 +1708,7 @@ async def test_image_capture_replays_the_existing_image_calculator( envelope: Final = database.rows[-1].envelope assert "private-image-content" not in envelope monkeypatch.delitem(litellm.model_cost, catalog_key) + monkeypatch.delitem(litellm.model_cost, tier_key, raising=False) replay: Final = price_usage(envelope) assert isinstance(replay, PricingSnapshot) assert float(replay.total_cost) == pytest.approx(source) From d8df34876ebcdbdafbf58188494b1ebc1eef119c Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:08:23 -0700 Subject: [PATCH 17/20] fix(accounting): retain input pixel rates for image replay --- litellm/types/utils.py | 1 + litellm/utils.py | 1 + .../spend_tracking/test_cost_compiler.py | 24 ++++++++++++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a5e2bb1eb68..e47eaa57112 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -437,6 +437,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x output output_cost_per_character_above_128k_tokens: float | None # only for vertex ai models output_cost_per_image: float | None + input_cost_per_pixel: ReadOnly[float | None] output_cost_per_pixel: ReadOnly[float | None] output_cost_per_image_token: float | None output_cost_per_image_token_batches: ReadOnly[float | None] diff --git a/litellm/utils.py b/litellm/utils.py index 5fb27ad6ab6..38494cd1d99 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7327,6 +7327,7 @@ def get_model_info_helper( output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), + input_cost_per_pixel=_model_info.get("input_cost_per_pixel", None), output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None), output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None), output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None), diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index 59be2d64d3f..b898012c57b 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1618,6 +1618,7 @@ async def test_live_adjustment_reload_is_captured_and_replay_remains_frozen(monk ("pixels", 2049, 0.6), ("tiered", 2048, 1.5), ("tiered-request", 2048, 1.5), + ("azure-tiered", 2048, 1.5), ), ) async def test_image_capture_replays_the_existing_image_calculator( @@ -1638,30 +1639,41 @@ async def test_image_capture_replays_the_existing_image_calculator( "pixels": "fal-ai/flux/dev", "tiered": "dall-e-capture", "tiered-request": "dall-e-capture", + "azure-tiered": "dall-e-3", }[pricing] - provider: Final = "fal_ai" if pricing == "pixels" else "openai" + provider: Final = "fal_ai" if pricing == "pixels" else "azure" if pricing == "azure-tiered" else "openai" + catalog_provider: Final = provider catalog_key: Final = f"{provider}/{model}" if pricing == "pixels" else model monkeypatch.setitem( litellm.model_cost, catalog_key, { - "litellm_provider": provider, + "litellm_provider": catalog_provider, "mode": "image_generation", **{ "tokens": {"input_cost_per_token": 0.01, "output_cost_per_image_token": 0.02}, "images": {"output_cost_per_image": 0.25}, "tiered": {"output_cost_per_image": 0.25}, "tiered-request": {"output_cost_per_image": 0.25}, + "azure-tiered": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, "pixels": {"output_cost_per_image": 0.1, "output_cost_per_pixel": 0.1 / (1024 * 1024)}, }[pricing], }, ) - tier_key: Final = f"hd/1024-x-1024/{model}" - if pricing.startswith("tiered"): + tier_key: Final = f"{provider}/hd/1024-x-1024/{model}" if pricing == "azure-tiered" else f"hd/1024-x-1024/{model}" + if "tiered" in pricing: monkeypatch.setitem( litellm.model_cost, tier_key, - {"litellm_provider": provider, "mode": "image_generation", "output_cost_per_image": 0.75}, + { + "litellm_provider": catalog_provider, + "mode": "image_generation", + **( + {"input_cost_per_pixel": 0.75 / (1024 * 1024)} + if pricing == "azure-tiered" + else {"output_cost_per_image": 0.75} + ), + }, ) logger: Final = _logger("aimage_generation") logger.model = model @@ -1687,7 +1699,7 @@ async def test_image_capture_replays_the_existing_image_calculator( ] * 2, "size": None if pricing == "tiered-request" else "1024x1024", - "quality": None if pricing == "tiered-request" else "hd" if pricing == "tiered" else "standard", + "quality": None if pricing == "tiered-request" else "hd" if "tiered" in pricing else "standard", "usage": { "input_tokens": 10, "output_tokens": 5, From 9586bb433650d298ec39a4861de8effd55ba6595 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:31:51 -0700 Subject: [PATCH 18/20] ci(accounting): report existing ledger integration coverage --- .circleci/scripts/run_integration.sh | 1 + .github/workflows/test-integration.yml | 14 ++++++++++++++ codecov.yaml | 2 ++ 3 files changed, 17 insertions(+) diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 1380b07de53..211572dd77f 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -256,6 +256,7 @@ fi env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ INTEGRATION_RUN_ID="$integration_identity" \ + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-}" \ DATABASE_URL="$DATABASE_URL" \ "${shared_services[@]}" "${worker_coverage[@]}" \ INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \ diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 73159bd64d2..9352482adc7 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -43,6 +43,9 @@ jobs: name: integration ${{ matrix.suite }} if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci' environment: integration-license + permissions: + contents: read + id-token: write runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }} timeout-minutes: 60 strategy: @@ -150,6 +153,7 @@ jobs: timeout-minutes: 45 env: SUITE: ${{ matrix.suite }} + PYTEST_ADDOPTS: ${{ matrix.suite == 'database' && '--cov=litellm.proxy.db.accounting_ledger --cov=litellm.proxy.spend_tracking.cost_compiler --cov-branch --cov-report=xml:test-results/accounting-coverage.xml' || '' }} INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }} INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }} LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }} @@ -159,6 +163,16 @@ jobs: *) bash .circleci/scripts/run_integration.sh "$SUITE" ;; esac + - name: Upload accounting ledger integration coverage + if: ${{ !cancelled() && matrix.suite == 'database' && hashFiles('test-results/accounting-coverage.xml') != '' }} + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 + with: + use_oidc: true + files: test-results/accounting-coverage.xml + disable_search: true + flags: accounting-ledger + fail_ci_if_error: true + - name: Verify every case ran exactly once if: ${{ !cancelled() && matrix.suite != 'browser' }} timeout-minutes: 2 diff --git a/codecov.yaml b/codecov.yaml index 521088868bd..d777602525d 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -20,6 +20,8 @@ flag_management: # the stale line maps mark comment lines of since-edited files as missed, # sinking patch coverage on unrelated PRs. individual_flags: + - name: accounting-ledger + carryforward: false - name: proxy-mgmt-behavior carryforward: false - name: security From 3e3b0d2d7e67b8cc762ed3511a908760c93e5863 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:42:30 -0700 Subject: [PATCH 19/20] test(accounting): isolate nested coverage reporting --- tests/integration/database/test_harness_lifecycle.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/database/test_harness_lifecycle.py b/tests/integration/database/test_harness_lifecycle.py index d600a92c5a2..f18cc9cf22c 100644 --- a/tests/integration/database/test_harness_lifecycle.py +++ b/tests/integration/database/test_harness_lifecycle.py @@ -81,6 +81,7 @@ def _nested_pytest_environment(environment: Mapping[str, str]) -> dict[str, str] "INTEGRATION_CASE_SHARD_INDEX", "CIRCLE_NODE_TOTAL", "CIRCLE_NODE_INDEX", + "PYTEST_ADDOPTS", "PYTEST_XDIST_WORKER", "PYTEST_XDIST_WORKER_COUNT", "PYTEST_XDIST_TESTRUNUID", @@ -107,8 +108,16 @@ def test_runner_environment_drops_inherited_provider_api_keys() -> None: @pytest.mark.parametrize("fails", (False, True), ids=("passing-test", "failing-test")) -def test_sdk_teardown_delivers_logging_before_restoring_callbacks(tmp_path: Path, fails: bool) -> None: +def test_sdk_teardown_delivers_logging_before_restoring_callbacks( + tmp_path: Path, fails: bool, monkeypatch: pytest.MonkeyPatch +) -> None: root: Final = Path(__file__).resolve().parents[3] + coverage_report: Final = tmp_path / "outer-coverage.xml" + coverage_report.write_text("parent coverage report") + monkeypatch.setenv( + "PYTEST_ADDOPTS", + f"--cov=litellm.proxy.spend_tracking.cost_compiler --cov-report=xml:{coverage_report}", + ) events: Final = tmp_path / "callbacks.json" probe: Final = tmp_path / "test_callback_teardown.py" probe.write_text( @@ -175,6 +184,7 @@ def test_sdk_teardown_delivers_logging_before_restoring_callbacks(tmp_path: Path check=False, ) assert result.returncode == (1 if fails else 0), result.stdout + result.stderr + assert coverage_report.read_text() == "parent coverage report", "nested pytest overwrote parent coverage" cases: Final = ElementTree.parse(tmp_path / "junit.xml").findall(".//testcase") assert tuple(case.attrib["name"] for case in cases) == ("test_request", "test_next") assert all(case.find("error") is None for case in cases), result.stdout + result.stderr From cb8fcaeee161563aa8938c386153d47fb9b5b081 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:53:11 -0700 Subject: [PATCH 20/20] fix(accounting): isolate fallback pricing by attempt --- .circleci/scripts/run_integration.sh | 1 - .github/workflows/test-integration.yml | 14 -- codecov.yaml | 2 - litellm/litellm_core_utils/litellm_logging.py | 5 +- litellm/main.py | 54 +++---- .../database/test_harness_lifecycle.py | 12 +- .../test_sagemaker_completion_handler.py | 13 +- .../spend_tracking/test_cost_compiler.py | 41 ++++++ .../test_register_model_custom_pricing.py | 138 ++++++++++++++++-- 9 files changed, 200 insertions(+), 80 deletions(-) diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 211572dd77f..1380b07de53 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -256,7 +256,6 @@ fi env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ INTEGRATION_RUN_ID="$integration_identity" \ - PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-}" \ DATABASE_URL="$DATABASE_URL" \ "${shared_services[@]}" "${worker_coverage[@]}" \ INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \ diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 9352482adc7..73159bd64d2 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -43,9 +43,6 @@ jobs: name: integration ${{ matrix.suite }} if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci' environment: integration-license - permissions: - contents: read - id-token: write runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }} timeout-minutes: 60 strategy: @@ -153,7 +150,6 @@ jobs: timeout-minutes: 45 env: SUITE: ${{ matrix.suite }} - PYTEST_ADDOPTS: ${{ matrix.suite == 'database' && '--cov=litellm.proxy.db.accounting_ledger --cov=litellm.proxy.spend_tracking.cost_compiler --cov-branch --cov-report=xml:test-results/accounting-coverage.xml' || '' }} INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }} INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }} LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }} @@ -163,16 +159,6 @@ jobs: *) bash .circleci/scripts/run_integration.sh "$SUITE" ;; esac - - name: Upload accounting ledger integration coverage - if: ${{ !cancelled() && matrix.suite == 'database' && hashFiles('test-results/accounting-coverage.xml') != '' }} - uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 - with: - use_oidc: true - files: test-results/accounting-coverage.xml - disable_search: true - flags: accounting-ledger - fail_ci_if_error: true - - name: Verify every case ran exactly once if: ${{ !cancelled() && matrix.suite != 'browser' }} timeout-minutes: 2 diff --git a/codecov.yaml b/codecov.yaml index d777602525d..521088868bd 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -20,8 +20,6 @@ flag_management: # the stale line maps mark comment lines of since-edited files as missed, # sinking patch coverage on unrelated PRs. individual_flags: - - name: accounting-ledger - carryforward: false - name: proxy-mgmt-behavior carryforward: false - name: security diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 30061050f66..41d95903785 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2154,11 +2154,14 @@ class Logging(LiteLLMLoggingBaseClass): self, model: str, provider: str, - overrides: Mapping[str, object], + overrides: Mapping[str, object] | None, deployment_id: str | None = None, ) -> None: from litellm.utils import model_info_with_pricing_overrides + if overrides is None: + self.request_pricing = None + return base: Final = self._request_pricing_model_info(model, provider, overrides, deployment_id) normalized: Final = _REQUEST_MODEL_INFO.validate_python( { diff --git a/litellm/main.py b/litellm/main.py index 170a4cd3551..d6706cdf99d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1480,10 +1480,16 @@ def _set_request_pricing( ) -> None: if not isinstance(logging_obj, LiteLLMLoggingObj): return + custom_pricing: Final = ( + (kwargs.get("input_cost_per_token") is not None and kwargs.get("output_cost_per_token") is not None) + or kwargs.get("input_cost_per_second") is not None + or kwargs.get("output_cost_per_second") is not None + or kwargs.get("cost_per_second") is not None + ) logging_obj.set_request_pricing( model, custom_llm_provider, - _build_custom_pricing_entry(custom_llm_provider, kwargs, model_info), + _build_custom_pricing_entry(custom_llm_provider, kwargs, model_info) if custom_pricing else None, _get_router_deployment_id(kwargs), ) @@ -5942,19 +5948,13 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _set_request_pricing( - logging_obj=litellm_logging_obj, - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) + _set_request_pricing( + logging_obj=litellm_logging_obj, + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6961,12 +6961,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -7013,19 +7007,13 @@ def embedding( **non_default_params, ) - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _set_request_pricing( - logging_obj=litellm_logging_obj, - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) + _set_request_pricing( + logging_obj=litellm_logging_obj, + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=kwargs.get("model_info"), + ) logging: Final[LiteLLMLoggingObj] = litellm_logging_obj logging.update_environment_variables( diff --git a/tests/integration/database/test_harness_lifecycle.py b/tests/integration/database/test_harness_lifecycle.py index f18cc9cf22c..d600a92c5a2 100644 --- a/tests/integration/database/test_harness_lifecycle.py +++ b/tests/integration/database/test_harness_lifecycle.py @@ -81,7 +81,6 @@ def _nested_pytest_environment(environment: Mapping[str, str]) -> dict[str, str] "INTEGRATION_CASE_SHARD_INDEX", "CIRCLE_NODE_TOTAL", "CIRCLE_NODE_INDEX", - "PYTEST_ADDOPTS", "PYTEST_XDIST_WORKER", "PYTEST_XDIST_WORKER_COUNT", "PYTEST_XDIST_TESTRUNUID", @@ -108,16 +107,8 @@ def test_runner_environment_drops_inherited_provider_api_keys() -> None: @pytest.mark.parametrize("fails", (False, True), ids=("passing-test", "failing-test")) -def test_sdk_teardown_delivers_logging_before_restoring_callbacks( - tmp_path: Path, fails: bool, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_sdk_teardown_delivers_logging_before_restoring_callbacks(tmp_path: Path, fails: bool) -> None: root: Final = Path(__file__).resolve().parents[3] - coverage_report: Final = tmp_path / "outer-coverage.xml" - coverage_report.write_text("parent coverage report") - monkeypatch.setenv( - "PYTEST_ADDOPTS", - f"--cov=litellm.proxy.spend_tracking.cost_compiler --cov-report=xml:{coverage_report}", - ) events: Final = tmp_path / "callbacks.json" probe: Final = tmp_path / "test_callback_teardown.py" probe.write_text( @@ -184,7 +175,6 @@ def test_sdk_teardown_delivers_logging_before_restoring_callbacks( check=False, ) assert result.returncode == (1 if fails else 0), result.stdout + result.stderr - assert coverage_report.read_text() == "parent coverage report", "nested pytest overwrote parent coverage" cases: Final = ElementTree.parse(tmp_path / "junit.xml").findall(".//testcase") assert tuple(case.attrib["name"] for case in cases) == ("test_request", "test_next") assert all(case.find("error") is None for case in cases), result.stdout + result.stderr diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 496eb5fe25b..4c253a10116 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -298,13 +298,8 @@ def test_installed_botocore_signs_the_completion_request(): from litellm.llms.sagemaker.completion.handler import SagemakerLLM request = SagemakerLLM()._prepare_request( - credentials=Credentials("test-key", "test-secret"), - model="test-endpoint", - data={"inputs": "ping"}, - messages=[], - litellm_params={}, - optional_params={}, - aws_region_name="us-west-2", + credentials=Credentials("test-key", "test-secret"), model="test-endpoint", data={"inputs": "ping"}, + messages=[], litellm_params={}, optional_params={}, aws_region_name="us-west-2", ) assert request.body == b'{"inputs": "ping"}' assert "/us-west-2/sagemaker/aws4_request" in request.headers["Authorization"] @@ -452,7 +447,9 @@ def test_completion_sagemaker_prompt_template_non_stream( }, ) ) - url: Final = "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/deepseek_coder_6.7_instruct/invocations" + url: Final = ( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/deepseek_coder_6.7_instruct/invocations" + ) route: Final = respx_mock.post(url).mock( return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) ) diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py index b898012c57b..077a7b8b379 100644 --- a/tests/unit/proxy/spend_tracking/test_cost_compiler.py +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -1726,3 +1726,44 @@ async def test_image_capture_replays_the_existing_image_calculator( assert float(replay.total_cost) == pytest.approx(source) finally: await meter.stop(1) + + +@pytest.mark.asyncio +async def test_delayed_attempt_capture_keeps_its_prices_after_fallback_reset(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.main import _set_request_pricing + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + catalog: Final = { + "openai/capture-model": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + } + monkeypatch.setattr(litellm, "model_cost", catalog) + monkeypatch.setattr(litellm, "cost_discount_config", {}) + monkeypatch.setattr(litellm, "cost_margin_config", {}) + logger: Final = _capture_logger() + failed_attempt: Final = logger.attempt_scoped_copy() + _set_request_pricing(logger, "capture-model", "openai", {}, None) + response: Final = ModelResponse(model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + for attempt, expected in ((logger, 0.018), (failed_attempt, 0.18)): + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + assert attempt.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(expected) + capture_logging_usage(meter, attempt, response, _AT, _AT) + await meter.close() + with monkeypatch.context() as changed_catalog: + changed_catalog.setattr(litellm, "model_cost", {}) + replay: Final = price_usage(database.rows[-1].envelope) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(expected) + finally: + await meter.stop(1) diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 5d77492ee58..d971f0d7014 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -227,7 +227,10 @@ def test_register_model_strips_none_litellm_provider(): registered = litellm.model_cost.get(model_key) assert registered is not None, f"{model_key} should be in model_cost" # The key may be absent entirely, but if present it must not be None. - assert "litellm_provider" not in registered or registered["litellm_provider"] is not None + assert ( + "litellm_provider" not in registered + or registered["litellm_provider"] is not None + ) # Downstream consumers must accept this entry for any provider, # mirroring what the cost calculator does. assert _check_provider_match(registered, "openai") is True @@ -285,7 +288,9 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp # The strip must have removed the None-valued provider that # ``get_model_info`` returned. The key may be absent entirely, but # it must never be present with value ``None``. - assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), ( + assert "litellm_provider" not in registered or ( + registered["litellm_provider"] is not None + ), ( "register_model failed to strip litellm_provider=None returned " f"by get_model_info, got {registered.get('litellm_provider')!r}" ) @@ -346,8 +351,14 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa ) registered = litellm.model_cost[registered_key] - assert registered.get("cache_creation_input_token_cost") == builtin["cache_creation_input_token_cost"] - assert registered.get("cache_read_input_token_cost") == builtin["cache_read_input_token_cost"] + assert ( + registered.get("cache_creation_input_token_cost") + == builtin["cache_creation_input_token_cost"] + ) + assert ( + registered.get("cache_read_input_token_cost") + == builtin["cache_read_input_token_cost"] + ) assert registered["litellm_provider"] == "bedrock" usage = Usage( @@ -411,7 +422,8 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog): ) assert any( - registered_key in record.message and "cache_creation_input_token_cost" in record.message + registered_key in record.message + and "cache_creation_input_token_cost" in record.message for record in caplog.records ), "expected a warning naming the unmapped key and the cache cost fields" finally: @@ -598,14 +610,17 @@ def test_register_model_router_add_deployment_custom_pricing_applies(): # the deployment's provider. from litellm.utils import _check_provider_match - registered_keys = [k for k in (deployment_model, model_key) if k in litellm.model_cost] + registered_keys = [ + k for k in (deployment_model, model_key) if k in litellm.model_cost + ] assert registered_keys, ( - f"Router.add_deployment did not register custom pricing for {model_key} / {deployment_model}" + "Router.add_deployment did not register custom pricing for " + f"{model_key} / {deployment_model}" ) for k in registered_keys: - assert _check_provider_match(litellm.model_cost[k], "openai") is True, ( - f"custom pricing for {k} was dropped by _check_provider_match" - ) + assert ( + _check_provider_match(litellm.model_cost[k], "openai") is True + ), f"custom pricing for {k} was dropped by _check_provider_match" finally: litellm.model_cost.pop(model_key, None) litellm.model_cost.pop(deployment_model, None) @@ -1870,3 +1885,106 @@ def test_sdk_followup_prices_change_only_after_explicit_registration(call_type: finally: _restore_model_cost_entries(original) litellm.open_ai_chat_completion_models.intersection_update(models_before) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ("completion", "acompletion", "embedding", "aembedding")) +@pytest.mark.parametrize("custom_primary", (True, False)) +async def test_router_fallback_prices_belong_to_the_served_attempt( + monkeypatch: pytest.MonkeyPatch, call_type: str, custom_primary: bool +) -> None: + import httpx + import respx + from datetime import datetime, timezone + + model: Final = "openai/fallback-price-regression" + embedding: Final = "embedding" in call_type + keys: Final = (model, model.removeprefix("openai/"), "price-primary", "price-fallback") + original: Final = _snapshot_model_cost_entries(keys) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "cost_discount_config", {}) + monkeypatch.setattr(litellm, "cost_margin_config", {}) + try: + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "embedding" if embedding else "chat", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + }, + persist_across_reloads=False, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": model, + "api_key": "fake-key", + "api_base": f"https://{name}.example/v1", + **({"input_cost_per_token": rate, "output_cost_per_token": rate} if rate is not None else {}), + }, + "model_info": {"id": f"price-{name}"}, + } + for name, rate in ( + ("primary", 0.0 if custom_primary else None), + ("fallback", None if custom_primary else 0.003), + ) + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + registered: Final = _snapshot_model_cost_entries(keys) + logger: Final = Logging( + model=model, + messages=[], + stream=False, + call_type=call_type, + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="fallback-pricing", + function_id="fallback-pricing", + ) + endpoint: Final = "embeddings" if embedding else "chat/completions" + body: Final = ( + { + "object": "list", + "model": model.removeprefix("openai/"), + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 10, "total_tokens": 10}, + } + if embedding + else { + "id": "fallback-result", + "object": "chat.completion", + "created": 1, + "model": model.removeprefix("openai/"), + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ) + with respx.mock(assert_all_called=True) as wire: + primary: Final = wire.post(f"https://primary.example/v1/{endpoint}").mock( + return_value=httpx.Response(429, json={"error": {"message": "rate limit", "type": "rate_limit_error"}}) + ) + fallback: Final = wire.post(f"https://fallback.example/v1/{endpoint}").mock( + return_value=httpx.Response(200, json=body) + ) + request: Final = {"model": "primary", "litellm_logging_obj": logger} + if call_type == "acompletion": + response = await router.acompletion(**request, messages=[{"role": "user", "content": "probe"}]) + elif call_type == "completion": + response = router.completion(**request, messages=[{"role": "user", "content": "probe"}]) + elif call_type == "aembedding": + response = await router.aembedding(**request, input=["probe"]) + else: + response = router.embedding(**request, input=["probe"]) + expected: Final = (0.01 if embedding else 0.02) if custom_primary else (0.03 if embedding else 0.045) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(expected) + assert response._hidden_params["response_cost"] == pytest.approx(expected) + assert primary.called + assert fallback.call_count == 1 + assert _snapshot_model_cost_entries(keys) == registered + finally: + _restore_model_cost_entries(original)