mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
feat(accounting): add accounting contracts, PricingSnapshot and ledger schema (LIT-10193) (#48862)
* feat(accounting): add typed contracts and spend log adapters * refactor(accounting): reuse spend identity constants and register tests * refactor(accounting): remove unused spend log conversion API * feat(accounting): add PricingSnapshot, delivery enums and projection dimensions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(db): add LiteLLM_AccountingEvent and LiteLLM_AccountingDelivery ledger tables Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(accounting): export JSON schema and semantic fixtures for the Rust cost crate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(accounting): drop the schema and fixture export, the Python package is the contract Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(accounting): make tests/unit/accounting a package Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(accounting): validate pricing snapshots and ledger rows * fix(otel): register the accounting ledger tables as span tables Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(accounting): address review findings on dimensions, token subsets, multipliers and agent target Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: joshua <joshua@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
cf7c35a117
commit
4516a251c9
14 changed files with 1695 additions and 0 deletions
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -137,6 +137,7 @@ jobs:
|
|||
include:
|
||||
- shard: core-utils
|
||||
test-path: |-
|
||||
tests/unit/accounting
|
||||
tests/unit/batch_completion
|
||||
tests/unit/decisions
|
||||
tests/unit/extraction
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
CREATE TYPE "AccountingEventKind" AS ENUM ('inbound_usage', 'outbound_usage', 'cost');
|
||||
|
||||
CREATE TYPE "AccountingDeliveryState" AS ENUM ('pending', 'applied', 'parked');
|
||||
|
||||
CREATE TYPE "AccountingDeliveryTarget" AS ENUM ('LiteLLM_VerificationToken', 'LiteLLM_UserTable', 'LiteLLM_TeamTable', 'LiteLLM_TeamMembership', 'LiteLLM_OrganizationTable', 'LiteLLM_OrganizationMembership', 'LiteLLM_EndUserTable', 'LiteLLM_AgentsTable', 'LiteLLM_TagTable', 'LiteLLM_ProjectTable', 'LiteLLM_ModelAccessGroupBudgetTable', 'LiteLLM_BudgetTable', 'LiteLLM_BudgetWindowSpend', 'LiteLLM_DailyUserSpend', 'LiteLLM_DailyTeamSpend', 'LiteLLM_DailyOrganizationSpend', 'LiteLLM_DailyEndUserSpend', 'LiteLLM_DailyAgentSpend', 'LiteLLM_DailyTagSpend', 'LiteLLM_DailyModelUsage', 'LiteLLM_DailyToolSpend', 'LiteLLM_AutoRouterDailySpend');
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingEvent" (
|
||||
"event_id" TEXT NOT NULL,
|
||||
"kind" "AccountingEventKind" NOT NULL,
|
||||
"source_event_id" TEXT,
|
||||
"schema_version" INTEGER NOT NULL,
|
||||
"payload_hash" TEXT NOT NULL,
|
||||
"occurred_at" TIMESTAMP(3) NOT NULL,
|
||||
"recorded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"payload" JSONB NOT NULL,
|
||||
|
||||
CONSTRAINT "LiteLLM_AccountingEvent_schema_version_check" CHECK ("schema_version" > 0),
|
||||
CONSTRAINT "LiteLLM_AccountingEvent_pkey" PRIMARY KEY ("event_id")
|
||||
);
|
||||
|
||||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingDelivery" (
|
||||
"event_id" TEXT NOT NULL,
|
||||
"target" "AccountingDeliveryTarget" NOT NULL,
|
||||
"version" INTEGER NOT NULL,
|
||||
"state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
|
||||
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"applied_at" TIMESTAMP(3),
|
||||
"last_error" TEXT,
|
||||
|
||||
CONSTRAINT "LiteLLM_AccountingDelivery_version_check" CHECK ("version" > 0),
|
||||
CONSTRAINT "LiteLLM_AccountingDelivery_attempts_check" CHECK ("attempts" >= 0),
|
||||
CONSTRAINT "LiteLLM_AccountingDelivery_pkey" PRIMARY KEY ("event_id","target","version")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_recorded_at_idx" ON "LiteLLM_AccountingEvent"("recorded_at");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_source_event_id_idx" ON "LiteLLM_AccountingEvent"("kind", "source_event_id");
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingDelivery_target_state_next_attempt_at_idx" ON "LiteLLM_AccountingDelivery"("target", "state", "next_attempt_at");
|
||||
|
|
@ -2061,3 +2061,73 @@ model LiteLLM_TelemetryReport {
|
|||
@@index([window_end])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
enum AccountingEventKind {
|
||||
inbound_usage
|
||||
outbound_usage
|
||||
cost
|
||||
}
|
||||
|
||||
enum AccountingDeliveryState {
|
||||
pending
|
||||
applied
|
||||
parked
|
||||
}
|
||||
|
||||
enum AccountingDeliveryTarget {
|
||||
LiteLLM_VerificationToken
|
||||
LiteLLM_UserTable
|
||||
LiteLLM_TeamTable
|
||||
LiteLLM_TeamMembership
|
||||
LiteLLM_OrganizationTable
|
||||
LiteLLM_OrganizationMembership
|
||||
LiteLLM_EndUserTable
|
||||
LiteLLM_AgentsTable
|
||||
LiteLLM_TagTable
|
||||
LiteLLM_ProjectTable
|
||||
LiteLLM_ModelAccessGroupBudgetTable
|
||||
LiteLLM_BudgetTable
|
||||
LiteLLM_BudgetWindowSpend
|
||||
LiteLLM_DailyUserSpend
|
||||
LiteLLM_DailyTeamSpend
|
||||
LiteLLM_DailyOrganizationSpend
|
||||
LiteLLM_DailyEndUserSpend
|
||||
LiteLLM_DailyAgentSpend
|
||||
LiteLLM_DailyTagSpend
|
||||
LiteLLM_DailyModelUsage
|
||||
LiteLLM_DailyToolSpend
|
||||
LiteLLM_AutoRouterDailySpend
|
||||
}
|
||||
|
||||
// Durable accounting ledger. One row per usage or cost event; payload holds the
|
||||
// private event envelope versioned by schema_version. Usage rows retain request-scoped
|
||||
// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
|
||||
model LiteLLM_AccountingEvent {
|
||||
event_id String @id
|
||||
kind AccountingEventKind
|
||||
source_event_id String?
|
||||
schema_version Int
|
||||
payload_hash String
|
||||
occurred_at DateTime
|
||||
recorded_at DateTime @default(now())
|
||||
payload Json
|
||||
|
||||
@@index([recorded_at])
|
||||
@@index([kind, source_event_id])
|
||||
}
|
||||
|
||||
// One row per (event, target table, version): pending work for the projector and,
|
||||
// once applied in the same transaction as the target write, the receipt that stops a retry.
|
||||
model LiteLLM_AccountingDelivery {
|
||||
event_id String
|
||||
target AccountingDeliveryTarget
|
||||
version Int
|
||||
state AccountingDeliveryState @default(pending)
|
||||
attempts Int @default(0)
|
||||
next_attempt_at DateTime @default(now())
|
||||
applied_at DateTime?
|
||||
last_error String?
|
||||
|
||||
@@id([event_id, target, version])
|
||||
@@index([target, state, next_attempt_at])
|
||||
}
|
||||
|
|
|
|||
62
litellm/accounting/__init__.py
Normal file
62
litellm/accounting/__init__.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
"""Public accounting contracts. Importing these values does not enable accounting."""
|
||||
|
||||
from litellm.accounting.measurements import measurements_for
|
||||
from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
|
||||
from litellm.accounting.types import (
|
||||
AdmissionDecision,
|
||||
AggregateQuery,
|
||||
AggregateRow,
|
||||
CostEvent,
|
||||
DeliveryState,
|
||||
DeliveryTarget,
|
||||
Dimension,
|
||||
DimensionFilter,
|
||||
DimensionName,
|
||||
EventKind,
|
||||
Granularity,
|
||||
InboundUsageEvent,
|
||||
Measurement,
|
||||
Metric,
|
||||
MetricName,
|
||||
Multiplier,
|
||||
NamedCost,
|
||||
OutboundUsageEvent,
|
||||
Outcome,
|
||||
PricedQuantity,
|
||||
PricingSnapshot,
|
||||
Rate,
|
||||
Unpriced,
|
||||
validate_aggregate_rows,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AdmissionDecision",
|
||||
"AggregateQuery",
|
||||
"AggregateRow",
|
||||
"BudgetEnforcer",
|
||||
"CostAggregator",
|
||||
"CostCompiler",
|
||||
"CostEvent",
|
||||
"DeliveryState",
|
||||
"DeliveryTarget",
|
||||
"Dimension",
|
||||
"DimensionFilter",
|
||||
"DimensionName",
|
||||
"EventKind",
|
||||
"Granularity",
|
||||
"InboundUsageEvent",
|
||||
"Measurement",
|
||||
"Metric",
|
||||
"MetricName",
|
||||
"Multiplier",
|
||||
"NamedCost",
|
||||
"OutboundUsageEvent",
|
||||
"Outcome",
|
||||
"PricedQuantity",
|
||||
"PricingSnapshot",
|
||||
"Rate",
|
||||
"Unpriced",
|
||||
"UsageMeter",
|
||||
"measurements_for",
|
||||
"validate_aggregate_rows",
|
||||
]
|
||||
13
litellm/accounting/measurements.py
Normal file
13
litellm/accounting/measurements.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
|
||||
|
||||
|
||||
def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
|
||||
"""Return the complete contribution set for one event, ready for one apply call."""
|
||||
metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
|
||||
occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
|
||||
return tuple(
|
||||
Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
|
||||
for metric in metrics or ()
|
||||
)
|
||||
85
litellm/accounting/protocols.py
Normal file
85
litellm/accounting/protocols.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
|
||||
|
||||
The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
|
||||
Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
|
||||
model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
|
||||
These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
|
||||
Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
|
||||
T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
|
||||
"""
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from litellm.accounting.types import (
|
||||
AdmissionDecision,
|
||||
AggregateQuery,
|
||||
AggregateRow,
|
||||
CostEvent,
|
||||
Dimension,
|
||||
InboundUsageEvent,
|
||||
Measurement,
|
||||
Metric,
|
||||
OutboundUsageEvent,
|
||||
Unpriced,
|
||||
)
|
||||
|
||||
|
||||
class UsageMeter(Protocol):
|
||||
async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
|
||||
"""Acknowledge commit, no-op identical IDs and reject conflicting content.
|
||||
|
||||
Persist the event and its request-scoped pricing context before acknowledging.
|
||||
HTTP acknowledgement is a separate policy. Unknown usage remains None.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class CostCompiler(Protocol):
|
||||
async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
|
||||
"""Reuse the committed charge or finalize one charge and its delivery intent.
|
||||
|
||||
Resolve captured request-scoped context by event identity and cost-map rates
|
||||
at compilation. Persist the resolved inputs in the required snapshot. Replay
|
||||
reuses the committed result and never consumes free allowance again.
|
||||
Missing pricing is Unpriced, never an inferred zero.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class CostAggregator(Protocol):
|
||||
async def apply(self, measurements: tuple[Measurement, ...]) -> None:
|
||||
"""Apply one source event's complete contribution set per target/version.
|
||||
|
||||
Each target commits all contributions and its receipt atomically. A receipt
|
||||
for the first metric must not suppress the rest. Usage need not wait for cost.
|
||||
"""
|
||||
...
|
||||
|
||||
async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
|
||||
"""Return disjoint [start, end) rows per group using existing daily projections.
|
||||
|
||||
Filters combine with AND, values within a filter with IN. DAY emits daily
|
||||
buckets; None emits whole-interval totals. Reject unsupported metric/scope
|
||||
combinations. Different groups may overlap, including non-additive tags.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class BudgetEnforcer(Protocol):
|
||||
async def reserve(
|
||||
self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
|
||||
) -> AdmissionDecision:
|
||||
"""Apply existing admission policies and estimates; duplicate IDs add no hold."""
|
||||
...
|
||||
|
||||
async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
|
||||
"""Settle the committed charge with atomic deduplication and hold mutation.
|
||||
|
||||
Preserve scope/window attribution and retry unfinished settlement after
|
||||
restart. SQL projection receipts do not provide live-counter atomicity.
|
||||
"""
|
||||
...
|
||||
|
||||
async def release(self, reservation_id: str) -> None:
|
||||
"""Idempotently release unused exposure at the request boundary, preserving charges."""
|
||||
...
|
||||
428
litellm/accounting/types.py
Normal file
428
litellm/accounting/types.py
Normal file
|
|
@ -0,0 +1,428 @@
|
|||
"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
|
||||
Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
|
||||
Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
|
||||
|
||||
|
||||
def _utc(value: datetime) -> datetime:
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
|
||||
|
||||
|
||||
class MetricName(str, Enum):
|
||||
"""Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
|
||||
|
||||
REQUESTS = "requests"
|
||||
SUCCESSFUL_REQUESTS = "successful_requests"
|
||||
FAILED_REQUESTS = "failed_requests"
|
||||
PROMPT_TOKENS = "prompt_tokens"
|
||||
COMPLETION_TOKENS = "completion_tokens"
|
||||
REASONING_TOKENS = "reasoning_tokens"
|
||||
TOTAL_TOKENS = "total_tokens"
|
||||
CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
|
||||
CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
|
||||
CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
|
||||
SPEND = "spend"
|
||||
|
||||
|
||||
class DimensionName(str, Enum):
|
||||
KEY = "key"
|
||||
USER = "user"
|
||||
TEAM = "team"
|
||||
ORGANIZATION = "organization"
|
||||
END_USER = "end_user"
|
||||
MODEL = "model"
|
||||
MODEL_GROUP = "model_group"
|
||||
DEPLOYMENT = "deployment"
|
||||
PROVIDER = "provider"
|
||||
TOOL = "tool"
|
||||
AGENT = "agent"
|
||||
PROJECT = "project"
|
||||
MODEL_ACCESS_GROUP = "model_access_group"
|
||||
API_BASE = "api_base"
|
||||
SERVICE_TIER = "service_tier"
|
||||
TAG = "tag"
|
||||
|
||||
|
||||
class Outcome(str, Enum):
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class Granularity(str, Enum):
|
||||
DAY = "day"
|
||||
|
||||
|
||||
class EventKind(str, Enum):
|
||||
INBOUND_USAGE = "inbound_usage"
|
||||
OUTBOUND_USAGE = "outbound_usage"
|
||||
COST = "cost"
|
||||
|
||||
|
||||
class DeliveryState(str, Enum):
|
||||
PENDING = "pending"
|
||||
APPLIED = "applied"
|
||||
PARKED = "parked"
|
||||
|
||||
|
||||
class DeliveryTarget(str, Enum):
|
||||
"""One member per existing table the projector writes; each is applied and receipted independently."""
|
||||
|
||||
KEY_SPEND = "LiteLLM_VerificationToken"
|
||||
USER_SPEND = "LiteLLM_UserTable"
|
||||
TEAM_SPEND = "LiteLLM_TeamTable"
|
||||
TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
|
||||
ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
|
||||
ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
|
||||
END_USER_SPEND = "LiteLLM_EndUserTable"
|
||||
AGENT_SPEND = "LiteLLM_AgentsTable"
|
||||
TAG_SPEND = "LiteLLM_TagTable"
|
||||
PROJECT_SPEND = "LiteLLM_ProjectTable"
|
||||
MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
|
||||
BUDGET_SPEND = "LiteLLM_BudgetTable"
|
||||
BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
|
||||
DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
|
||||
DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
|
||||
DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
|
||||
DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
|
||||
DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
|
||||
DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
|
||||
DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
|
||||
DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
|
||||
AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
|
||||
|
||||
|
||||
_SUBSET_PAIRS: Final = (
|
||||
(MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
|
||||
(MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
|
||||
)
|
||||
|
||||
|
||||
def _check_subsets(values: Mapping[str, Decimal]) -> None:
|
||||
for part, whole in _SUBSET_PAIRS:
|
||||
if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
|
||||
raise ValueError(f"{part.value} cannot exceed {whole.value}")
|
||||
|
||||
|
||||
class Metric(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
name: MetricName
|
||||
value: Quantity
|
||||
unit: Literal["count", "USD"] = "count"
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_unit(self) -> Self:
|
||||
if (self.name is MetricName.SPEND) != (self.unit == "USD"):
|
||||
raise ValueError("SPEND requires USD; usage metrics require count")
|
||||
if self.unit == "count" and self.value != self.value.to_integral_value():
|
||||
raise ValueError("Count metrics require integral values")
|
||||
return self
|
||||
|
||||
|
||||
class Dimension(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
name: DimensionName
|
||||
value: Identifier
|
||||
|
||||
|
||||
_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
|
||||
|
||||
|
||||
def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
|
||||
if len(frozenset(values)) != len(values):
|
||||
raise ValueError("Duplicate dimension values")
|
||||
single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
|
||||
if len(frozenset(single_names)) != len(single_names):
|
||||
raise ValueError("Only tag and model access group dimensions may have multiple values")
|
||||
return values
|
||||
|
||||
|
||||
Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
|
||||
_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
|
||||
|
||||
|
||||
def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
|
||||
if len(frozenset(value.name for value in values)) != len(values):
|
||||
raise ValueError("Duplicate metric names")
|
||||
return values
|
||||
|
||||
|
||||
Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
|
||||
|
||||
|
||||
def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
|
||||
if end < start or (end == start and not allow_empty):
|
||||
raise ValueError("End must follow start")
|
||||
|
||||
|
||||
class InboundUsageEvent(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
event_id: Identifier
|
||||
request_id: Identifier
|
||||
call_type: CallTypes
|
||||
started_at: Instant
|
||||
ended_at: Instant
|
||||
outcome: Outcome
|
||||
dimensions: Dimensions
|
||||
metrics: Metrics
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_caller_activity(self) -> Self:
|
||||
_interval(self.started_at, self.ended_at, allow_empty=True)
|
||||
if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
|
||||
raise ValueError("Inbound events own only caller request metrics")
|
||||
if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
|
||||
raise ValueError("Inbound events count exactly one caller request")
|
||||
for metric in self.metrics:
|
||||
if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
|
||||
raise ValueError("Successful request count must match outcome")
|
||||
if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
|
||||
raise ValueError("Failed request count must match outcome")
|
||||
return self
|
||||
|
||||
|
||||
class OutboundUsageEvent(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
event_id: Identifier
|
||||
attempt_id: Identifier
|
||||
request_id: Identifier | None
|
||||
retry_of_attempt_id: Identifier | None
|
||||
call_type: CallTypes
|
||||
started_at: Instant
|
||||
ended_at: Instant
|
||||
outcome: Outcome
|
||||
dimensions: Dimensions
|
||||
metrics: Metrics | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_attempt_usage(self) -> Self:
|
||||
_interval(self.started_at, self.ended_at, allow_empty=True)
|
||||
if self.retry_of_attempt_id == self.attempt_id:
|
||||
raise ValueError("An attempt cannot retry itself")
|
||||
if self.metrics is not None and any(
|
||||
metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
|
||||
):
|
||||
raise ValueError("Outbound events own only attempt usage metrics")
|
||||
if self.metrics is not None:
|
||||
_check_subsets({metric.name.value: metric.value for metric in self.metrics})
|
||||
return self
|
||||
|
||||
|
||||
PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
|
||||
|
||||
|
||||
class Rate(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
measure: Identifier
|
||||
unit: PricingUnit
|
||||
usd_per_unit: Quantity
|
||||
|
||||
|
||||
class PricedQuantity(BaseModel):
|
||||
"""A pricing input in its billing unit. Only seconds may be fractional."""
|
||||
|
||||
model_config = _CONFIG
|
||||
|
||||
measure: Identifier
|
||||
unit: PricingUnit
|
||||
value: Quantity
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_discrete_quantity(self) -> Self:
|
||||
if self.unit != "second" and self.value != self.value.to_integral_value():
|
||||
raise ValueError("Discrete priced quantities require integral values")
|
||||
return self
|
||||
|
||||
|
||||
class Multiplier(BaseModel):
|
||||
"""A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
|
||||
|
||||
model_config = _CONFIG
|
||||
|
||||
name: Identifier
|
||||
value: Identifier
|
||||
factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
|
||||
|
||||
|
||||
class NamedCost(BaseModel):
|
||||
"""A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
|
||||
|
||||
model_config = _CONFIG
|
||||
|
||||
name: Identifier
|
||||
amount: Quantity
|
||||
|
||||
|
||||
class PricingSnapshot(BaseModel):
|
||||
"""Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
|
||||
|
||||
Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
|
||||
resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
|
||||
Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
|
||||
Component sums and float tolerances belong to reconciliation, not this structural contract.
|
||||
"""
|
||||
|
||||
model_config = _CONFIG
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
calculator_version: Identifier
|
||||
model: Identifier
|
||||
provider: Identifier | None
|
||||
pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
|
||||
priced_model_key: Identifier
|
||||
multipliers: tuple[Multiplier, ...] = ()
|
||||
rates: tuple[Rate, ...] = ()
|
||||
usage: tuple[PricedQuantity, ...] = ()
|
||||
input_cost: Quantity | None = None
|
||||
output_cost: Quantity | None = None
|
||||
cache_read_cost: Quantity | None = None
|
||||
cache_creation_cost: Quantity | None = None
|
||||
reasoning_cost: Quantity | None = None
|
||||
tool_usage_cost: Quantity | None = None
|
||||
guardrail_cost: Quantity | None = None
|
||||
additional_costs: tuple[NamedCost, ...] = ()
|
||||
original_cost: Quantity | None = None
|
||||
discount_percent: Quantity | None = None
|
||||
discount_amount: Quantity | None = None
|
||||
margin_percent: Quantity | None = None
|
||||
margin_fixed_amount: Quantity | None = None
|
||||
margin_total_amount: Quantity | None = None
|
||||
upstream_reported_cost: Quantity | None = None
|
||||
total_cost: Quantity
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_snapshot(self) -> Self:
|
||||
rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
|
||||
quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
|
||||
if len(rate_keys) != len(self.rates):
|
||||
raise ValueError("Duplicate rate measures and units")
|
||||
if len(quantity_keys) != len(self.usage):
|
||||
raise ValueError("Duplicate priced quantity measures and units")
|
||||
if not rate_keys.issubset(quantity_keys):
|
||||
raise ValueError("Every rate requires one priced quantity with the same measure and unit")
|
||||
if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
|
||||
raise ValueError("Duplicate additional cost names")
|
||||
if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
|
||||
raise ValueError("Duplicate multiplier names")
|
||||
_check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
|
||||
return self
|
||||
|
||||
|
||||
class CostEvent(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
event_id: Identifier
|
||||
usage_event_id: Identifier
|
||||
occurred_at: Instant
|
||||
dimensions: Dimensions
|
||||
charges: Metrics
|
||||
snapshot: PricingSnapshot
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_charge(self) -> Self:
|
||||
if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
|
||||
raise ValueError("A cost event requires one authoritative USD spend metric")
|
||||
if self.event_id == self.usage_event_id:
|
||||
raise ValueError("Cost and usage events require distinct identities")
|
||||
if self.snapshot.total_cost != self.charges[0].value:
|
||||
raise ValueError("Snapshot total must equal the authoritative spend")
|
||||
return self
|
||||
|
||||
|
||||
class Unpriced(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
usage_event_id: Identifier
|
||||
reason: Identifier
|
||||
|
||||
|
||||
class Measurement(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
source_event_id: Identifier
|
||||
occurred_at: Instant
|
||||
dimensions: Dimensions
|
||||
metric: Metric
|
||||
|
||||
|
||||
class DimensionFilter(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
name: DimensionName
|
||||
values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
|
||||
|
||||
|
||||
class AggregateQuery(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
metric: MetricName
|
||||
filters: tuple[DimensionFilter, ...]
|
||||
group_by: tuple[DimensionName, ...]
|
||||
start: Instant
|
||||
end: Instant
|
||||
granularity: Granularity | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_query(self) -> Self:
|
||||
_interval(self.start, self.end)
|
||||
if len(frozenset(item.name for item in self.filters)) != len(self.filters):
|
||||
raise ValueError("Use one IN filter per dimension; filters combine with AND")
|
||||
if len(frozenset(self.group_by)) != len(self.group_by):
|
||||
raise ValueError("Duplicate group-by dimensions")
|
||||
if any(
|
||||
(instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
|
||||
for instant in (self.start, self.end)
|
||||
):
|
||||
raise ValueError("Initial reporting queries require UTC day boundaries")
|
||||
return self
|
||||
|
||||
|
||||
class AggregateRow(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
start: Instant
|
||||
end: Instant
|
||||
dimensions: Dimensions
|
||||
value: Quantity
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_bounds(self) -> Self:
|
||||
_interval(self.start, self.end)
|
||||
return self
|
||||
|
||||
|
||||
def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
|
||||
"""Validate [start, end) buckets per group; overlapping tag groups are not additive."""
|
||||
groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
|
||||
for group in groups:
|
||||
ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
|
||||
if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
|
||||
raise ValueError("Aggregate buckets overlap within a dimension group")
|
||||
return rows
|
||||
|
||||
|
||||
class AdmissionDecision(BaseModel):
|
||||
model_config = _CONFIG
|
||||
|
||||
allowed: bool
|
||||
reason: Identifier | None = None
|
||||
|
|
@ -338,6 +338,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
|
|||
"LiteLLM_ToolTable",
|
||||
"LiteLLM_TelemetryReport",
|
||||
"LiteLLM_AccessGroupTable",
|
||||
"LiteLLM_AccountingDelivery",
|
||||
"LiteLLM_AccountingEvent",
|
||||
"LiteLLM_ClaudeCodePluginTable",
|
||||
"LiteLLM_MemoryTable",
|
||||
"LiteLLM_AdaptiveRouterState",
|
||||
|
|
|
|||
|
|
@ -2061,3 +2061,73 @@ model LiteLLM_TelemetryReport {
|
|||
@@index([window_end])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
enum AccountingEventKind {
|
||||
inbound_usage
|
||||
outbound_usage
|
||||
cost
|
||||
}
|
||||
|
||||
enum AccountingDeliveryState {
|
||||
pending
|
||||
applied
|
||||
parked
|
||||
}
|
||||
|
||||
enum AccountingDeliveryTarget {
|
||||
LiteLLM_VerificationToken
|
||||
LiteLLM_UserTable
|
||||
LiteLLM_TeamTable
|
||||
LiteLLM_TeamMembership
|
||||
LiteLLM_OrganizationTable
|
||||
LiteLLM_OrganizationMembership
|
||||
LiteLLM_EndUserTable
|
||||
LiteLLM_AgentsTable
|
||||
LiteLLM_TagTable
|
||||
LiteLLM_ProjectTable
|
||||
LiteLLM_ModelAccessGroupBudgetTable
|
||||
LiteLLM_BudgetTable
|
||||
LiteLLM_BudgetWindowSpend
|
||||
LiteLLM_DailyUserSpend
|
||||
LiteLLM_DailyTeamSpend
|
||||
LiteLLM_DailyOrganizationSpend
|
||||
LiteLLM_DailyEndUserSpend
|
||||
LiteLLM_DailyAgentSpend
|
||||
LiteLLM_DailyTagSpend
|
||||
LiteLLM_DailyModelUsage
|
||||
LiteLLM_DailyToolSpend
|
||||
LiteLLM_AutoRouterDailySpend
|
||||
}
|
||||
|
||||
// Durable accounting ledger. One row per usage or cost event; payload holds the
|
||||
// private event envelope versioned by schema_version. Usage rows retain request-scoped
|
||||
// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
|
||||
model LiteLLM_AccountingEvent {
|
||||
event_id String @id
|
||||
kind AccountingEventKind
|
||||
source_event_id String?
|
||||
schema_version Int
|
||||
payload_hash String
|
||||
occurred_at DateTime
|
||||
recorded_at DateTime @default(now())
|
||||
payload Json
|
||||
|
||||
@@index([recorded_at])
|
||||
@@index([kind, source_event_id])
|
||||
}
|
||||
|
||||
// One row per (event, target table, version): pending work for the projector and,
|
||||
// once applied in the same transaction as the target write, the receipt that stops a retry.
|
||||
model LiteLLM_AccountingDelivery {
|
||||
event_id String
|
||||
target AccountingDeliveryTarget
|
||||
version Int
|
||||
state AccountingDeliveryState @default(pending)
|
||||
attempts Int @default(0)
|
||||
next_attempt_at DateTime @default(now())
|
||||
applied_at DateTime?
|
||||
last_error String?
|
||||
|
||||
@@id([event_id, target, version])
|
||||
@@index([target, state, next_attempt_at])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2061,3 +2061,73 @@ model LiteLLM_TelemetryReport {
|
|||
@@index([window_end])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
enum AccountingEventKind {
|
||||
inbound_usage
|
||||
outbound_usage
|
||||
cost
|
||||
}
|
||||
|
||||
enum AccountingDeliveryState {
|
||||
pending
|
||||
applied
|
||||
parked
|
||||
}
|
||||
|
||||
enum AccountingDeliveryTarget {
|
||||
LiteLLM_VerificationToken
|
||||
LiteLLM_UserTable
|
||||
LiteLLM_TeamTable
|
||||
LiteLLM_TeamMembership
|
||||
LiteLLM_OrganizationTable
|
||||
LiteLLM_OrganizationMembership
|
||||
LiteLLM_EndUserTable
|
||||
LiteLLM_AgentsTable
|
||||
LiteLLM_TagTable
|
||||
LiteLLM_ProjectTable
|
||||
LiteLLM_ModelAccessGroupBudgetTable
|
||||
LiteLLM_BudgetTable
|
||||
LiteLLM_BudgetWindowSpend
|
||||
LiteLLM_DailyUserSpend
|
||||
LiteLLM_DailyTeamSpend
|
||||
LiteLLM_DailyOrganizationSpend
|
||||
LiteLLM_DailyEndUserSpend
|
||||
LiteLLM_DailyAgentSpend
|
||||
LiteLLM_DailyTagSpend
|
||||
LiteLLM_DailyModelUsage
|
||||
LiteLLM_DailyToolSpend
|
||||
LiteLLM_AutoRouterDailySpend
|
||||
}
|
||||
|
||||
// Durable accounting ledger. One row per usage or cost event; payload holds the
|
||||
// private event envelope versioned by schema_version. Usage rows retain request-scoped
|
||||
// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
|
||||
model LiteLLM_AccountingEvent {
|
||||
event_id String @id
|
||||
kind AccountingEventKind
|
||||
source_event_id String?
|
||||
schema_version Int
|
||||
payload_hash String
|
||||
occurred_at DateTime
|
||||
recorded_at DateTime @default(now())
|
||||
payload Json
|
||||
|
||||
@@index([recorded_at])
|
||||
@@index([kind, source_event_id])
|
||||
}
|
||||
|
||||
// One row per (event, target table, version): pending work for the projector and,
|
||||
// once applied in the same transaction as the target write, the receipt that stops a retry.
|
||||
model LiteLLM_AccountingDelivery {
|
||||
event_id String
|
||||
target AccountingDeliveryTarget
|
||||
version Int
|
||||
state AccountingDeliveryState @default(pending)
|
||||
attempts Int @default(0)
|
||||
next_attempt_at DateTime @default(now())
|
||||
applied_at DateTime?
|
||||
last_error String?
|
||||
|
||||
@@id([event_id, target, version])
|
||||
@@index([target, state, next_attempt_at])
|
||||
}
|
||||
|
|
|
|||
128
tests/integration/database/test_accounting_ledger.py
Normal file
128
tests/integration/database/test_accounting_ledger.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from psycopg import sql
|
||||
|
||||
from litellm.accounting import DeliveryState, DeliveryTarget, EventKind
|
||||
from tests.integration._support.database import scratch_database
|
||||
|
||||
_SCHEMA: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/schema.prisma"
|
||||
_EVENT_INSERT: Final = (
|
||||
'INSERT INTO "LiteLLM_AccountingEvent" '
|
||||
"(event_id, kind, schema_version, payload_hash, occurred_at, payload) "
|
||||
"VALUES ('event', %s, %s, 'hash', '2026-01-01', '{}') RETURNING kind::text, schema_version"
|
||||
)
|
||||
_DELIVERY_INSERT: Final = (
|
||||
'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state, attempts) '
|
||||
"VALUES ('event', %s, %s, %s, %s) RETURNING target::text, version, state::text, attempts"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ledger_database() -> Iterator[str]:
|
||||
with scratch_database("accounting") as database_url:
|
||||
result: Final = subprocess.run(
|
||||
[sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(_SCHEMA)],
|
||||
env={**os.environ, "DATABASE_URL": database_url},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
yield database_url
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ledger(ledger_database: str) -> Iterator[psycopg.Connection[tuple[object, ...]]]:
|
||||
with psycopg.connect(ledger_database) as connection:
|
||||
try:
|
||||
yield connection
|
||||
finally:
|
||||
connection.rollback()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", tuple(EventKind))
|
||||
def test_ledger_accepts_every_public_event_kind(
|
||||
ledger: psycopg.Connection[tuple[object, ...]], kind: EventKind
|
||||
) -> None:
|
||||
row: Final = ledger.execute(_EVENT_INSERT, (kind.value, 1)).fetchone()
|
||||
assert row == (kind.value, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", tuple(DeliveryTarget))
|
||||
@pytest.mark.parametrize("state", tuple(DeliveryState))
|
||||
def test_delivery_accepts_every_public_target_and_state(
|
||||
ledger: psycopg.Connection[tuple[object, ...]], target: DeliveryTarget, state: DeliveryState
|
||||
) -> None:
|
||||
row: Final = ledger.execute(_DELIVERY_INSERT, (target.value, 1, state.value, 0)).fetchone()
|
||||
assert row == (target.value, 1, state.value, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enum_name", "values"),
|
||||
[
|
||||
("AccountingEventKind", tuple(value.value for value in EventKind)),
|
||||
("AccountingDeliveryState", tuple(value.value for value in DeliveryState)),
|
||||
("AccountingDeliveryTarget", tuple(value.value for value in DeliveryTarget)),
|
||||
],
|
||||
)
|
||||
def test_migrated_enums_match_the_public_contract(
|
||||
ledger: psycopg.Connection[tuple[object, ...]], enum_name: str, values: tuple[str, ...]
|
||||
) -> None:
|
||||
rows: Final = ledger.execute(
|
||||
"SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid WHERE t.typname = %s",
|
||||
(enum_name,),
|
||||
).fetchall()
|
||||
assert frozenset(row[0] for row in rows) == frozenset(values)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [0, -1])
|
||||
def test_event_rejects_nonpositive_schema_versions(
|
||||
ledger: psycopg.Connection[tuple[object, ...]], version: int
|
||||
) -> None:
|
||||
with pytest.raises(psycopg.errors.CheckViolation, match="AccountingEvent_schema_version_check"):
|
||||
ledger.execute(_EVENT_INSERT, (EventKind.OUTBOUND_USAGE.value, version))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("version", "attempts", "constraint"),
|
||||
[
|
||||
(0, 0, "AccountingDelivery_version_check"),
|
||||
(-1, 0, "AccountingDelivery_version_check"),
|
||||
(1, -1, "AccountingDelivery_attempts_check"),
|
||||
],
|
||||
)
|
||||
def test_delivery_rejects_invalid_versions_and_attempts(
|
||||
ledger: psycopg.Connection[tuple[object, ...]], version: int, attempts: int, constraint: str
|
||||
) -> None:
|
||||
with pytest.raises(psycopg.errors.CheckViolation, match=constraint):
|
||||
ledger.execute(_DELIVERY_INSERT, (DeliveryTarget.KEY_SPEND.value, version, "pending", attempts))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("column", ["state", "target"])
|
||||
def test_delivery_rejects_unknown_enum_values(ledger: psycopg.Connection[tuple[object, ...]], column: str) -> None:
|
||||
ledger.execute(_DELIVERY_INSERT, (DeliveryTarget.KEY_SPEND.value, 1, "pending", 0))
|
||||
with pytest.raises(psycopg.errors.InvalidTextRepresentation, match="invalid input value for enum"):
|
||||
ledger.execute(
|
||||
sql.SQL('UPDATE "LiteLLM_AccountingDelivery" SET {} = %s').format(sql.Identifier(column)), ("invalid",)
|
||||
)
|
||||
|
||||
|
||||
def test_event_rejects_unknown_kind(ledger: psycopg.Connection[tuple[object, ...]]) -> None:
|
||||
with pytest.raises(psycopg.errors.InvalidTextRepresentation, match="invalid input value for enum"):
|
||||
ledger.execute(_EVENT_INSERT, ("invalid", 1))
|
||||
|
||||
|
||||
def test_new_delivery_defaults_to_pending_with_no_attempts(ledger: psycopg.Connection[tuple[object, ...]]) -> None:
|
||||
row: Final = ledger.execute(
|
||||
'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) '
|
||||
"VALUES ('event', %s, 1) RETURNING state::text, attempts",
|
||||
(DeliveryTarget.KEY_SPEND.value,),
|
||||
).fetchone()
|
||||
assert row == ("pending", 0)
|
||||
0
tests/unit/accounting/__init__.py
Normal file
0
tests/unit/accounting/__init__.py
Normal file
235
tests/unit/accounting/test_measurements.py
Normal file
235
tests/unit/accounting/test_measurements.py
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.accounting import (
|
||||
CostEvent,
|
||||
Dimension,
|
||||
DimensionName,
|
||||
InboundUsageEvent,
|
||||
Measurement,
|
||||
Metric,
|
||||
MetricName,
|
||||
Outcome,
|
||||
OutboundUsageEvent,
|
||||
PricingSnapshot,
|
||||
measurements_for,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
_START: Final = datetime(2026, 1, 1, 23, 59, 59, tzinfo=timezone.utc)
|
||||
_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc)
|
||||
_DIMENSIONS: Final = (
|
||||
Dimension(name=DimensionName.TEAM, value="team"),
|
||||
Dimension(name=DimensionName.TAG, value="alpha"),
|
||||
Dimension(name=DimensionName.TAG, value="beta"),
|
||||
)
|
||||
|
||||
|
||||
def _snapshot(amount: Decimal) -> PricingSnapshot:
|
||||
return PricingSnapshot(
|
||||
calculator_version="test",
|
||||
model="test-model",
|
||||
provider=None,
|
||||
pricing_source="upstream_header",
|
||||
priced_model_key="test-model",
|
||||
upstream_reported_cost=amount,
|
||||
total_cost=amount,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("outcome", "outcome_metric"),
|
||||
[
|
||||
(Outcome.SUCCEEDED, MetricName.SUCCESSFUL_REQUESTS),
|
||||
(Outcome.FAILED, MetricName.FAILED_REQUESTS),
|
||||
(Outcome.CANCELLED, MetricName.FAILED_REQUESTS),
|
||||
],
|
||||
)
|
||||
def test_caller_measurements_keep_both_request_and_outcome_counts(outcome, outcome_metric):
|
||||
event = InboundUsageEvent(
|
||||
event_id="caller-event",
|
||||
request_id="caller",
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_END,
|
||||
outcome=outcome,
|
||||
dimensions=_DIMENSIONS,
|
||||
metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)), Metric(name=outcome_metric, value=Decimal(1))),
|
||||
)
|
||||
assert measurements_for(event) == (
|
||||
Measurement(
|
||||
source_event_id="caller-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.REQUESTS, value=Decimal(1)),
|
||||
),
|
||||
Measurement(
|
||||
source_event_id="caller-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=outcome_metric, value=Decimal(1)),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_usage_measurements_keep_every_metric_in_the_start_time_bucket():
|
||||
event = OutboundUsageEvent(
|
||||
event_id="usage-event",
|
||||
attempt_id="attempt",
|
||||
request_id="caller",
|
||||
retry_of_attempt_id=None,
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_END,
|
||||
outcome=Outcome.SUCCEEDED,
|
||||
dimensions=_DIMENSIONS,
|
||||
metrics=(
|
||||
Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)),
|
||||
Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)),
|
||||
Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),
|
||||
Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)),
|
||||
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)),
|
||||
),
|
||||
)
|
||||
assert measurements_for(event) == (
|
||||
Measurement(
|
||||
source_event_id="usage-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)),
|
||||
),
|
||||
Measurement(
|
||||
source_event_id="usage-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)),
|
||||
),
|
||||
Measurement(
|
||||
source_event_id="usage-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),
|
||||
),
|
||||
Measurement(
|
||||
source_event_id="usage-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)),
|
||||
),
|
||||
Measurement(
|
||||
source_event_id="usage-event",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metrics", [None, ()])
|
||||
def test_unknown_or_empty_usage_does_not_fabricate_zero_measurements(metrics):
|
||||
event = OutboundUsageEvent(
|
||||
event_id="usage-event",
|
||||
attempt_id="attempt",
|
||||
request_id=None,
|
||||
retry_of_attempt_id=None,
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_END,
|
||||
outcome=Outcome.SUCCEEDED,
|
||||
dimensions=(),
|
||||
metrics=metrics,
|
||||
)
|
||||
assert measurements_for(event) == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("amount", [Decimal(0), Decimal("0.1234567890123456789")])
|
||||
def test_cost_measurements_preserve_authoritative_amount_identity_and_occurrence(amount):
|
||||
event = CostEvent(
|
||||
event_id="cost-event",
|
||||
usage_event_id="usage-event",
|
||||
occurred_at=_END,
|
||||
dimensions=_DIMENSIONS,
|
||||
charges=(Metric(name=MetricName.SPEND, value=amount, unit="USD"),),
|
||||
snapshot=_snapshot(amount),
|
||||
)
|
||||
assert measurements_for(event) == (
|
||||
Measurement(
|
||||
source_event_id="cost-event",
|
||||
occurred_at=_END,
|
||||
dimensions=_DIMENSIONS,
|
||||
metric=Metric(name=MetricName.SPEND, value=amount, unit="USD"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_billed_retry_and_success_contribute_one_caller_request_and_both_charges():
|
||||
caller = InboundUsageEvent(
|
||||
event_id="caller-event",
|
||||
request_id="caller",
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_END,
|
||||
outcome=Outcome.SUCCEEDED,
|
||||
dimensions=_DIMENSIONS,
|
||||
metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),),
|
||||
)
|
||||
retry = OutboundUsageEvent(
|
||||
event_id="retry-usage",
|
||||
attempt_id="retry",
|
||||
request_id="caller",
|
||||
retry_of_attempt_id=None,
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_END,
|
||||
outcome=Outcome.FAILED,
|
||||
dimensions=_DIMENSIONS,
|
||||
metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(4)),),
|
||||
)
|
||||
success = OutboundUsageEvent(
|
||||
event_id="success-usage",
|
||||
attempt_id="success",
|
||||
request_id="caller",
|
||||
retry_of_attempt_id="retry",
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_END,
|
||||
outcome=Outcome.SUCCEEDED,
|
||||
dimensions=_DIMENSIONS,
|
||||
metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),),
|
||||
)
|
||||
retry_cost = CostEvent(
|
||||
event_id="retry-cost",
|
||||
usage_event_id="retry-usage",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
charges=(Metric(name=MetricName.SPEND, value=Decimal("0.025"), unit="USD"),),
|
||||
snapshot=_snapshot(Decimal("0.025")),
|
||||
)
|
||||
success_cost = CostEvent(
|
||||
event_id="success-cost",
|
||||
usage_event_id="success-usage",
|
||||
occurred_at=_START,
|
||||
dimensions=_DIMENSIONS,
|
||||
charges=(Metric(name=MetricName.SPEND, value=Decimal("0.125"), unit="USD"),),
|
||||
snapshot=_snapshot(Decimal("0.125")),
|
||||
)
|
||||
measurements = (
|
||||
*measurements_for(caller),
|
||||
*measurements_for(retry),
|
||||
*measurements_for(success),
|
||||
*measurements_for(retry_cost),
|
||||
*measurements_for(success_cost),
|
||||
)
|
||||
assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.REQUESTS) == Decimal(1)
|
||||
assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.TOTAL_TOKENS) == Decimal(16)
|
||||
assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.SPEND) == Decimal("0.150")
|
||||
assert {m.source_event_id for m in measurements} == {
|
||||
"caller-event",
|
||||
"retry-usage",
|
||||
"success-usage",
|
||||
"retry-cost",
|
||||
"success-cost",
|
||||
}
|
||||
484
tests/unit/accounting/test_types.py
Normal file
484
tests/unit/accounting/test_types.py
Normal file
|
|
@ -0,0 +1,484 @@
|
|||
import re
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.accounting import (
|
||||
AdmissionDecision,
|
||||
AggregateQuery,
|
||||
AggregateRow,
|
||||
CostEvent,
|
||||
DeliveryState,
|
||||
DeliveryTarget,
|
||||
Dimension,
|
||||
DimensionFilter,
|
||||
DimensionName,
|
||||
EventKind,
|
||||
Granularity,
|
||||
InboundUsageEvent,
|
||||
Metric,
|
||||
MetricName,
|
||||
Multiplier,
|
||||
Outcome,
|
||||
OutboundUsageEvent,
|
||||
PricedQuantity,
|
||||
PricingSnapshot,
|
||||
Rate,
|
||||
validate_aggregate_rows,
|
||||
)
|
||||
from litellm.accounting.types import PricingUnit
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc)
|
||||
_REQUEST: Final = Metric(name=MetricName.REQUESTS, value=Decimal(1))
|
||||
_TOKEN: Final = Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10))
|
||||
_SPEND: Final = Metric(name=MetricName.SPEND, value=Decimal("0.1234567890123456789"), unit="USD")
|
||||
_TEAM: Final = Dimension(name=DimensionName.TEAM, value="team")
|
||||
|
||||
|
||||
def _inbound(metrics=(_REQUEST,), outcome=Outcome.SUCCEEDED):
|
||||
return InboundUsageEvent(
|
||||
event_id="inbound:request",
|
||||
request_id="request",
|
||||
call_type=CallTypes.acompletion,
|
||||
started_at=_START,
|
||||
ended_at=_START,
|
||||
outcome=outcome,
|
||||
dimensions=(_TEAM,),
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
def _outbound(metrics=(_TOKEN,), **changes):
|
||||
return OutboundUsageEvent.model_validate(
|
||||
{
|
||||
"event_id": "outbound:attempt",
|
||||
"attempt_id": "attempt",
|
||||
"request_id": "request",
|
||||
"retry_of_attempt_id": None,
|
||||
"call_type": CallTypes.acompletion,
|
||||
"started_at": _START,
|
||||
"ended_at": _END,
|
||||
"outcome": Outcome.SUCCEEDED,
|
||||
"dimensions": (_TEAM,),
|
||||
"metrics": metrics,
|
||||
**changes,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _query(**changes):
|
||||
return AggregateQuery.model_validate(
|
||||
{
|
||||
"metric": MetricName.SPEND,
|
||||
"filters": (DimensionFilter(name=DimensionName.TEAM, values=("one", "two")),),
|
||||
"group_by": (DimensionName.MODEL,),
|
||||
"start": _START,
|
||||
"end": _END,
|
||||
"granularity": Granularity.DAY,
|
||||
**changes,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_wire_values_preserve_decimal_precision_and_normalize_aware_times():
|
||||
offset = timezone(timedelta(hours=-8))
|
||||
event = CostEvent(
|
||||
event_id="cost:attempt",
|
||||
usage_event_id="outbound:attempt",
|
||||
occurred_at=datetime(2025, 12, 31, 16, tzinfo=offset),
|
||||
dimensions=(_TEAM,),
|
||||
charges=(_SPEND,),
|
||||
snapshot=_snapshot(),
|
||||
)
|
||||
wire = event.model_dump(mode="json")
|
||||
assert wire["occurred_at"] == "2026-01-01T00:00:00Z"
|
||||
assert wire["charges"] == [{"name": "spend", "value": "0.1234567890123456789", "unit": "USD"}]
|
||||
assert event.occurred_at == _START
|
||||
assert CostEvent.model_validate_json(event.model_dump_json()).charges[0].value == _SPEND.value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "value", "unit", "message"),
|
||||
[
|
||||
(MetricName.SPEND, Decimal(1), "count", "SPEND requires USD"),
|
||||
(MetricName.REQUESTS, Decimal(1), "USD", "SPEND requires USD"),
|
||||
(MetricName.PROMPT_TOKENS, Decimal("1.5"), "count", "integral"),
|
||||
(MetricName.SPEND, Decimal("NaN"), "USD", "finite"),
|
||||
(MetricName.SPEND, Decimal("Infinity"), "USD", "finite"),
|
||||
(MetricName.SPEND, Decimal(-1), "USD", "greater than or equal"),
|
||||
(MetricName.SPEND, 0.1, "USD", "instance of Decimal"),
|
||||
],
|
||||
)
|
||||
def test_metrics_reject_wrong_units_nonfinite_values_and_binary_float_inputs(name, value, unit, message):
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
Metric(name=name, value=value, unit=unit)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metrics", [(_TOKEN,), (_SPEND,), ()])
|
||||
def test_inbound_activity_cannot_own_usage_or_cost_or_omit_request_count(metrics):
|
||||
with pytest.raises(ValidationError, match="Inbound events"):
|
||||
_inbound(metrics)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("outcome", "metric"),
|
||||
[
|
||||
(Outcome.FAILED, Metric(name=MetricName.SUCCESSFUL_REQUESTS, value=Decimal(1))),
|
||||
(Outcome.SUCCEEDED, Metric(name=MetricName.FAILED_REQUESTS, value=Decimal(1))),
|
||||
],
|
||||
)
|
||||
def test_inbound_outcome_counts_cannot_contradict_outcome(outcome, metric):
|
||||
with pytest.raises(ValidationError, match="count must match outcome"):
|
||||
_inbound((_REQUEST, metric), outcome)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metrics", [(_REQUEST,), (_SPEND,)])
|
||||
def test_outbound_attempts_cannot_own_caller_counts_or_charges(metrics):
|
||||
with pytest.raises(ValidationError, match="Outbound events own only"):
|
||||
_outbound(metrics)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "message"),
|
||||
[
|
||||
({"started_at": datetime(2026, 1, 1)}, "timezone"),
|
||||
({"ended_at": _START - timedelta(seconds=1)}, "End must follow start"),
|
||||
({"retry_of_attempt_id": "attempt"}, "cannot retry itself"),
|
||||
({"dimensions": (_TEAM, _TEAM)}, "Duplicate dimension values"),
|
||||
({"dimensions": (_TEAM, Dimension(name=DimensionName.TEAM, value="other"))}, "Only tag and model access group"),
|
||||
(
|
||||
{
|
||||
"metrics": (
|
||||
Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(5)),
|
||||
Metric(name=MetricName.REASONING_TOKENS, value=Decimal(6)),
|
||||
)
|
||||
},
|
||||
"reasoning_tokens cannot exceed completion_tokens",
|
||||
),
|
||||
(
|
||||
{
|
||||
"metrics": (
|
||||
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(5)),
|
||||
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(6)),
|
||||
)
|
||||
},
|
||||
"cache_creation_input_tokens_above_1hr cannot exceed cache_creation_input_tokens",
|
||||
),
|
||||
({"event_id": " "}, "pattern"),
|
||||
({"metrics": (_TOKEN, _TOKEN)}, "Duplicate metric names"),
|
||||
],
|
||||
)
|
||||
def test_outbound_events_reject_invalid_identity_time_and_duplicate_attribution(changes, message):
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
_outbound(**changes)
|
||||
|
||||
|
||||
def test_unknown_and_known_zero_usage_are_distinct_and_values_are_immutable():
|
||||
unknown = _outbound(None)
|
||||
zero = _outbound((Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(0)),))
|
||||
assert unknown.metrics is None
|
||||
assert zero.metrics is not None and zero.metrics[0].value == 0
|
||||
assert _inbound().started_at == _inbound().ended_at
|
||||
with pytest.raises(ValidationError, match="frozen"):
|
||||
zero.attempt_id = "different"
|
||||
with pytest.raises(ValidationError, match="frozen"):
|
||||
zero.metrics[0].value = Decimal(10)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("charges", [(), (_TOKEN,), (_SPEND, _SPEND)])
|
||||
def test_cost_events_require_one_authoritative_spend(charges):
|
||||
with pytest.raises(ValidationError, match=r"authoritative USD spend|Duplicate metric"):
|
||||
CostEvent(
|
||||
event_id="cost",
|
||||
usage_event_id="usage",
|
||||
occurred_at=_START,
|
||||
dimensions=(),
|
||||
charges=charges,
|
||||
snapshot=_snapshot(),
|
||||
)
|
||||
|
||||
|
||||
def test_cost_event_cannot_reuse_its_usage_identity():
|
||||
with pytest.raises(ValidationError, match="distinct identities"):
|
||||
CostEvent(
|
||||
event_id="usage",
|
||||
usage_event_id="usage",
|
||||
occurred_at=_START,
|
||||
dimensions=(),
|
||||
charges=(_SPEND,),
|
||||
snapshot=_snapshot(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("granularity", [Granularity.DAY, None])
|
||||
def test_daily_and_whole_interval_queries_preserve_filter_and_group_semantics(granularity):
|
||||
query = _query(granularity=granularity)
|
||||
assert query.granularity is granularity
|
||||
assert query.filters[0].values == ("one", "two")
|
||||
assert query.group_by == (DimensionName.MODEL,)
|
||||
assert query.start == _START and query.end == _END
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "message"),
|
||||
[
|
||||
({"end": _START}, "End must follow start"),
|
||||
({"start": _START + timedelta(hours=1)}, "UTC day boundaries"),
|
||||
({"end": _END + timedelta(microseconds=1)}, "UTC day boundaries"),
|
||||
({"granularity": "hour"}, "day"),
|
||||
({"group_by": (DimensionName.TEAM, DimensionName.TEAM)}, "Duplicate group-by"),
|
||||
({"filters": (DimensionFilter(name=DimensionName.TEAM, values=("one",)),) * 2}, "one IN filter"),
|
||||
({"filters": ({"name": "team", "values": ()},)}, "at least 1"),
|
||||
],
|
||||
)
|
||||
def test_queries_reject_unsupported_or_ambiguous_ranges_and_filters(changes, message):
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
_query(**changes)
|
||||
|
||||
|
||||
def test_rows_allow_adjacent_buckets_and_overlapping_distinct_groups():
|
||||
first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM,), value=Decimal(1))
|
||||
second = AggregateRow(start=_END, end=_END + timedelta(days=1), dimensions=(_TEAM,), value=Decimal(2))
|
||||
other = AggregateRow(start=_START, end=_END, dimensions=(), value=Decimal(3))
|
||||
rows = (second, other, first)
|
||||
assert validate_aggregate_rows(rows) is rows
|
||||
assert validate_aggregate_rows(()) == ()
|
||||
|
||||
|
||||
def test_rows_reject_overlap_within_a_group_regardless_of_dimension_order():
|
||||
model = Dimension(name=DimensionName.MODEL, value="model")
|
||||
first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM, model), value=Decimal(1))
|
||||
overlapping = AggregateRow(start=_START, end=_END, dimensions=(model, _TEAM), value=Decimal(2))
|
||||
with pytest.raises(ValueError, match="overlap within a dimension group"):
|
||||
validate_aggregate_rows((first, overlapping))
|
||||
|
||||
|
||||
def test_rows_reject_empty_intervals():
|
||||
with pytest.raises(ValidationError, match="End must follow start"):
|
||||
AggregateRow(start=_START, end=_START, dimensions=(), value=Decimal(0))
|
||||
|
||||
|
||||
def test_admission_decision_preserves_a_denial_reason():
|
||||
decision = AdmissionDecision(allowed=False, reason="Team budget exhausted")
|
||||
assert not decision.allowed
|
||||
assert decision.reason == "Team budget exhausted"
|
||||
|
||||
|
||||
def _snapshot(**changes: object) -> PricingSnapshot:
|
||||
return PricingSnapshot.model_validate(
|
||||
{
|
||||
"calculator_version": "1.80.0",
|
||||
"model": "gpt-4o",
|
||||
"provider": "openai",
|
||||
"pricing_source": "cost_map",
|
||||
"priced_model_key": "openai/gpt-4o",
|
||||
"multipliers": (Multiplier(name="data_residency", value="eu", factor=Decimal("1.1")),),
|
||||
"rates": (Rate(measure="input", unit="token", usd_per_unit=Decimal("0.0000025")),),
|
||||
"usage": (PricedQuantity(measure="input", unit="token", value=_TOKEN.value),),
|
||||
"input_cost": Decimal("0.000025"),
|
||||
"total_cost": _SPEND.value,
|
||||
**changes,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_cost_event_carries_a_snapshot_whose_total_matches_the_spend():
|
||||
event = CostEvent(
|
||||
event_id="cost:attempt",
|
||||
usage_event_id="outbound:attempt",
|
||||
occurred_at=_START,
|
||||
dimensions=(_TEAM,),
|
||||
charges=(_SPEND,),
|
||||
snapshot=_snapshot(),
|
||||
)
|
||||
assert event.snapshot is not None
|
||||
assert event.snapshot.total_cost == _SPEND.value
|
||||
assert event.snapshot.multipliers[0].factor == Decimal("1.1")
|
||||
with pytest.raises(ValidationError, match="Snapshot total must equal the authoritative spend"):
|
||||
CostEvent(
|
||||
event_id="cost:attempt",
|
||||
usage_event_id="outbound:attempt",
|
||||
occurred_at=_START,
|
||||
dimensions=(_TEAM,),
|
||||
charges=(_SPEND,),
|
||||
snapshot=_snapshot(total_cost=Decimal("0.5")),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("changes", "message"),
|
||||
(
|
||||
(
|
||||
{
|
||||
"rates": (
|
||||
Rate(measure="input", unit="token", usd_per_unit=Decimal(1)),
|
||||
Rate(measure="input", unit="token", usd_per_unit=Decimal(2)),
|
||||
)
|
||||
},
|
||||
"Duplicate rate measures and units",
|
||||
),
|
||||
({"usage": (_SPEND,)}, "PricedQuantity"),
|
||||
({"usage": (_REQUEST,)}, "PricedQuantity"),
|
||||
({"pricing_source": "guess"}, "pricing_source"),
|
||||
({"multipliers": ({"name": "x", "value": "y", "factor": Decimal(0)},)}, "factor"),
|
||||
({"schema_version": 2}, "schema_version"),
|
||||
(
|
||||
{"additional_costs": ({"name": "fee", "amount": Decimal(1)}, {"name": "fee", "amount": Decimal(2)})},
|
||||
"Duplicate additional cost names",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_snapshots_reject_ambiguous_rates_wrong_usage_and_unknown_sources(changes: Mapping[str, object], message: str):
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
_snapshot(**changes)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("unit", "value"),
|
||||
[("token", "10"), ("second", "1.25"), ("character", "25"), ("image", "2"), ("request", "1")],
|
||||
)
|
||||
def test_snapshot_preserves_matching_rates_and_quantities_for_each_billing_unit(unit: PricingUnit, value: str) -> None:
|
||||
quantity: Final = PricedQuantity(measure="input", unit=unit, value=Decimal(value))
|
||||
rate: Final = Rate(measure="input", unit=unit, usd_per_unit=Decimal("0.003"))
|
||||
amount: Final = quantity.value * rate.usd_per_unit
|
||||
snapshot: Final = _snapshot(rates=(rate,), usage=(quantity,), total_cost=amount)
|
||||
restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json())
|
||||
assert restored == snapshot
|
||||
assert restored.usage[0].value * restored.rates[0].usd_per_unit == amount
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("usage", "message"),
|
||||
[
|
||||
((), "Every rate requires"),
|
||||
(({"measure": "output", "unit": "token", "value": Decimal(10)},), "Every rate requires"),
|
||||
(({"measure": "input", "unit": "second", "value": Decimal(10)},), "Every rate requires"),
|
||||
(
|
||||
({"measure": "input", "unit": "token", "value": Decimal(10)},) * 2,
|
||||
"Duplicate priced quantity",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_snapshot_rejects_missing_mismatched_or_duplicate_priced_quantities(
|
||||
usage: tuple[Mapping[str, object], ...], message: str
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
_snapshot(usage=usage)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("unit", ["token", "character", "image", "request"])
|
||||
def test_priced_counts_remain_integral(unit: PricingUnit) -> None:
|
||||
with pytest.raises(ValidationError, match="integral"):
|
||||
PricedQuantity(measure="input", unit=unit, value=Decimal("1.25"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "message"),
|
||||
[
|
||||
(Decimal(-1), "greater than or equal"),
|
||||
(Decimal("NaN"), "finite"),
|
||||
(Decimal("Infinity"), "finite"),
|
||||
(1.25, "instance of Decimal"),
|
||||
],
|
||||
)
|
||||
def test_pricing_values_reject_invalid_amounts(value: Decimal | float, message: str) -> None:
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
PricedQuantity.model_validate({"measure": "input", "unit": "second", "value": value})
|
||||
with pytest.raises(ValidationError, match=message):
|
||||
Rate.model_validate({"measure": "input", "unit": "second", "usd_per_unit": value})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("snapshot_fields", [{}, {"snapshot": None}])
|
||||
def test_cost_event_cannot_omit_its_derivation(snapshot_fields: Mapping[str, object]) -> None:
|
||||
with pytest.raises(ValidationError, match="snapshot"):
|
||||
CostEvent.model_validate(
|
||||
{
|
||||
"event_id": "cost",
|
||||
"usage_event_id": "usage",
|
||||
"occurred_at": _START,
|
||||
"dimensions": (),
|
||||
"charges": (_SPEND,),
|
||||
**snapshot_fields,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_upstream_charge_snapshot_preserves_reported_cost_without_inventing_rates() -> None:
|
||||
snapshot: Final = _snapshot(
|
||||
pricing_source="upstream_header", rates=(), usage=(), upstream_reported_cost=_SPEND.value
|
||||
)
|
||||
restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json())
|
||||
assert restored.upstream_reported_cost == restored.total_cost == _SPEND.value
|
||||
assert restored.rates == restored.usage == ()
|
||||
|
||||
|
||||
def test_subset_token_counts_round_trip_without_changing_totals() -> None:
|
||||
metrics: Final = (
|
||||
Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(10)),
|
||||
Metric(name=MetricName.REASONING_TOKENS, value=Decimal(3)),
|
||||
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(8)),
|
||||
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(2)),
|
||||
)
|
||||
event: Final = _outbound(metrics=metrics)
|
||||
assert OutboundUsageEvent.model_validate_json(event.model_dump_json()).metrics == metrics
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"schema_path",
|
||||
[
|
||||
"schema.prisma",
|
||||
"litellm/proxy/schema.prisma",
|
||||
"litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("enum_name", "python_enum"),
|
||||
[
|
||||
("AccountingEventKind", EventKind),
|
||||
("AccountingDeliveryState", DeliveryState),
|
||||
("AccountingDeliveryTarget", DeliveryTarget),
|
||||
],
|
||||
)
|
||||
def test_prisma_accepts_exactly_the_public_accounting_enum_values(
|
||||
schema_path: str, enum_name: str, python_enum: type[Enum]
|
||||
) -> None:
|
||||
schema: Final = (Path(__file__).resolve().parents[3] / schema_path).read_text()
|
||||
declaration: Final = re.search(r"enum " + enum_name + r"\s*\{([^}]+)\}", schema)
|
||||
assert declaration is not None, f"Missing persisted enum {enum_name}"
|
||||
assert frozenset(declaration[1].split()) == frozenset(member.value for member in python_enum)
|
||||
|
||||
|
||||
def test_one_event_carries_every_matched_model_access_group():
|
||||
groups = (
|
||||
Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="gpt-group"),
|
||||
Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="claude-group"),
|
||||
)
|
||||
assert _outbound(dimensions=(_TEAM, *groups)).dimensions == (_TEAM, *groups)
|
||||
with pytest.raises(ValidationError, match="Duplicate dimension values"):
|
||||
_outbound(dimensions=(groups[0], groups[0]))
|
||||
|
||||
|
||||
def test_snapshots_reject_duplicate_multipliers_and_contradictory_token_counts():
|
||||
multiplier = Multiplier(name="data_residency", value="eu", factor=Decimal("1.1"))
|
||||
with pytest.raises(ValidationError, match="Duplicate multiplier names"):
|
||||
_snapshot(multipliers=(multiplier, multiplier))
|
||||
with pytest.raises(ValidationError, match="reasoning_tokens cannot exceed completion_tokens"):
|
||||
_snapshot(
|
||||
usage=(
|
||||
PricedQuantity(measure="input", unit="token", value=_TOKEN.value),
|
||||
PricedQuantity(measure="completion_tokens", unit="token", value=Decimal(5)),
|
||||
PricedQuantity(measure="reasoning_tokens", unit="token", value=Decimal(6)),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_agent_live_spend_is_a_delivery_target():
|
||||
assert DeliveryTarget.AGENT_SPEND.value == "LiteLLM_AgentsTable"
|
||||
Loading…
Add table
Reference in a new issue