scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge remote-tracking branch 'origin/main' into beluga_decrease_anys_20261102

Browse source
This commit is contained in:
mateo-berri 2026-11-04 20:19:09 -08:00
commit b471e57be8
389 changed files with 19535 additions and 6637 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql Normal file
View file

@ -0,0 +1,19 @@
SET LOCAL lock_timeout = '10s';
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" (
"delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
"request_id" TEXT NOT NULL,
"state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
"attempts" INTEGER NOT NULL DEFAULT 0,
"next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"applied_at" TIMESTAMP(3),
"last_error" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"accounting_payload" JSONB NOT NULL,
"is_batch" BOOLEAN NOT NULL
);
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_next_attempt_at_idx"
ON "LiteLLM_SpendLogDelivery" ("state", "next_attempt_at");
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_applied_at_idx"
ON "LiteLLM_SpendLogDelivery" ("state", "applied_at");
CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_request_id_idx"
ON "LiteLLM_SpendLogDelivery" ("request_id");

17
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -2141,6 +2141,23 @@ model LiteLLM_AccountingDelivery {
@@index([target, state, next_attempt_at])
}
model LiteLLM_SpendLogDelivery {
delivery_id String @id @default(dbgenerated("gen_random_uuid()::text"))
request_id String
state AccountingDeliveryState @default(pending)
accounting_payload Json
is_batch Boolean
attempts Int @default(0)
next_attempt_at DateTime @default(now())
applied_at DateTime?
last_error String?
created_at DateTime @default(now())
@@index([state, next_attempt_at])
@@index([state, applied_at])
@@index([request_id])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String

59
litellm/accounting/__init__.py
View file

@ -1,59 +0,0 @@
"""Public accounting contracts. Importing these values does not enable accounting."""
from litellm.accounting.measurements import measurements_for
from litellm.accounting.protocols import CostAggregator
from litellm.accounting.types import (
AdmissionDecision,
AggregateQuery,
AggregateRow,
CostEvent,
DeliveryState,
DeliveryTarget,
Dimension,
DimensionFilter,
DimensionName,
EventKind,
Granularity,
InboundUsageEvent,
Measurement,
Metric,
MetricName,
Multiplier,
NamedCost,
OutboundUsageEvent,
Outcome,
PricedQuantity,
PricingSnapshot,
Rate,
Unpriced,
validate_aggregate_rows,
)
__all__ = [
"AdmissionDecision",
"AggregateQuery",
"AggregateRow",
"CostAggregator",
"CostEvent",
"DeliveryState",
"DeliveryTarget",
"Dimension",
"DimensionFilter",
"DimensionName",
"EventKind",
"Granularity",
"InboundUsageEvent",
"Measurement",
"Metric",
"MetricName",
"Multiplier",
"NamedCost",
"OutboundUsageEvent",
"Outcome",
"PricedQuantity",
"PricingSnapshot",
"Rate",
"Unpriced",
"measurements_for",
"validate_aggregate_rows",
]

71
litellm/accounting/application.py Normal file
View file

@ -0,0 +1,71 @@
from typing import TYPE_CHECKING, Final
from litellm.accounting.delivery_projection import entity_transactions, project_deliveries
from litellm.accounting.projection import Contribution
from litellm.proxy.db.budget_window_spend_writer import commit_window_spend_updates
from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key
from litellm.proxy.db.db_span import db_span
from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction
from litellm.proxy.db.model_usage_rollup import queue_model_usage_upsert
from litellm.repositories.accounting_delivery_repository import DeliveryProjection, LedgerTransaction
if TYPE_CHECKING:
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.utils import PrismaClient, ProxyLogging
async def apply_deliveries(
transaction: LedgerTransaction,
deliveries: tuple[DeliveryProjection, ...],
*,
prisma_client: "PrismaClient",
writer: "DBSpendUpdateWriter",
proxy_logging_obj: "ProxyLogging",
) -> tuple[Contribution, ...]:
from litellm.proxy.guardrails.usage_tracking import UsageUnitIncrement, UsageUnitKey, upsert_usage_unit_row
contributions: Final = project_deliveries(deliveries)
await writer.apply_accounting_entities(
prisma_client, proxy_logging_obj, transaction, entity_transactions(contributions)
)
for table in sorted(DAILY_SPEND_TABLES.values(), key=lambda item: item.name):
rows = tuple(
{**dict(item.key), **dict(item.values), **dict(item.increments)}
for item in contributions
if item.table == table.name
)
merged = merge_by_conflict_key(table, rows)
for offset in range(0, len(merged), 100):
sql, params = build_bulk_upsert(table, merged[offset : offset + 100])
async with db_span("upsert_daily_spend", table.name):
await transaction.execute_raw(sql, *params)
async with transaction.batch_() as batcher:
for item in contributions:
if item.table == "LiteLLM_DailyModelUsage":
queue_model_usage_upsert(batcher, dict(item.key), dict(item.increments))
for item in contributions:
if item.table == "LiteLLM_DailyGuardrailUsageUnits":
key = dict(item.key)
counters = dict(item.increments)
await upsert_usage_unit_row(
prisma_client,
UsageUnitKey(key["guardrail_id"], key["date"], key["team_id"], key["api_key"], key["usage_unit"]),
UsageUnitIncrement(int(counters["units"]), float(counters["cost"]), int(counters["untracked_units"])),
outer_transaction=transaction,
)
windows: Final = tuple(_window(item) for item in contributions if item.table == "LiteLLM_BudgetWindowSpend")
await commit_window_spend_updates(prisma_client, windows, outer_transaction=transaction)
return contributions
def _window(item: Contribution) -> WindowSpendTransaction:
key: Final = dict(item.key)
values: Final = dict(item.values)
return WindowSpendTransaction(
entity_type=key["entity_type"],
entity_id=key["entity_id"],
window_duration=key["window_duration"],
window_start=values["window_start"] or "",
spend=float(dict(item.increments)["spend"]),
started_at=values["started_at"],
)

78
litellm/accounting/delivery_projection.py Normal file
View file

@ -0,0 +1,78 @@
from collections.abc import Iterable
from itertools import chain, groupby
from typing import TYPE_CHECKING, Final
from urllib.parse import quote
from litellm.accounting.projection import Contribution, project
from litellm.repositories.accounting_delivery_repository import DeliveryProjection
def _destination(contribution: Contribution) -> tuple[str, tuple[tuple[str, str], ...], str]:
return (
contribution.table,
contribution.key,
(dict(contribution.values).get("window_start") or "")
if contribution.table == "LiteLLM_BudgetWindowSpend"
else "",
)
def _sum(contributions: tuple[Contribution, ...]) -> Contribution:
first: Final = contributions[0]
increments: Final = tuple(
(column, sum(dict(item.increments).get(column, 0) for item in contributions))
for column in dict.fromkeys(
column for column, _ in chain.from_iterable(item.increments for item in contributions)
)
)
starts: Final = tuple(
value
for column, value in chain.from_iterable(item.values for item in contributions)
if column == "started_at" and value
)
values: Final = tuple(
(column, min(starts) if column == "started_at" and starts else value) for column, value in first.values
)
return Contribution(first.table, first.key, increments, values)
def project_deliveries(deliveries: Iterable[DeliveryProjection]) -> tuple[Contribution, ...]:
ordered: Final = sorted(chain.from_iterable(project(delivery.row) for delivery in deliveries), key=_destination)
return tuple(_sum(tuple(group)) for _, group in groupby(ordered, key=_destination))
def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpdateTransactions":
return {
"user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_UserTable")),
"end_user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_EndUserTable")),
"key_list_transactions": dict(_entity_totals(contributions, "LiteLLM_VerificationToken")),
"team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamTable")),
"team_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamMembership")),
"org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationTable")),
"org_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationMembership")),
"project_list_transactions": dict(_entity_totals(contributions, "LiteLLM_ProjectTable")),
"tag_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TagTable")),
"agent_list_transactions": dict(_entity_totals(contributions, "LiteLLM_AgentsTable")),
"model_access_group_list_transactions": dict(
_entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable")
),
}
def _entity_totals(contributions: tuple[Contribution, ...], table: str) -> tuple[tuple[str, float], ...]:
return tuple(
(_entity_key(item), float(dict(item.increments)["spend"])) for item in contributions if item.table == table
)
def _entity_key(item: Contribution) -> str:
fields: Final = dict(item.key)
if item.table == "LiteLLM_TeamMembership":
return f"team_id::{fields['team_id']}::user_id::{fields['user_id']}"
if item.table == "LiteLLM_OrganizationMembership":
return f"org_id::{quote(fields['organization_id'], safe='')}::user_id::{quote(fields['user_id'], safe='')}"
return item.key[0][1]
if TYPE_CHECKING:
from litellm.proxy._types import DBSpendUpdateTransactions

13
litellm/accounting/measurements.py
View file

@ -1,13 +0,0 @@
from typing import Final
from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
"""Return the complete contribution set for one event, ready for one apply call."""
metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
return tuple(
Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
for metric in metrics or ()
)

30
litellm/accounting/metrics.py Normal file
View file

@ -0,0 +1,30 @@
from typing import Final
from prometheus_client import Gauge, Histogram
from litellm.repositories.accounting_delivery_repository import DeliveryMetrics
_PENDING: Final = Gauge("litellm_accounting_pending", "Pending SpendLogs deliveries", multiprocess_mode="livemax")
_PARKED: Final = Gauge("litellm_accounting_parked", "Parked SpendLogs deliveries", multiprocess_mode="livemax")
_OLDEST: Final = Gauge(
"litellm_accounting_oldest_pending_seconds",
"Age of the oldest pending SpendLogs delivery",
multiprocess_mode="livemax",
)
_BATCH_TIME: Final = Histogram("litellm_accounting_batch_seconds", "SpendLogs delivery worker batch time")
_BATCH_COUNT: Final = Gauge(
"litellm_accounting_batch_deliveries", "Retained batch deliveries", multiprocess_mode="livemax"
)
_TABLE_BYTES: Final = Gauge(
"litellm_accounting_delivery_table_bytes", "Delivery table size in bytes", multiprocess_mode="livemax"
)
def record_deliveries(metrics: DeliveryMetrics, seconds: float) -> None:
_PENDING.set(metrics.pending_count)
_PARKED.set(metrics.parked_count)
_OLDEST.set(max(0.0, metrics.oldest_pending_seconds))
_BATCH_TIME.observe(seconds)
_BATCH_COUNT.set(metrics.batch_count)
_TABLE_BYTES.set(metrics.table_bytes)

28
litellm/accounting/projection.py
View file

@ -1,5 +1,5 @@
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
@ -24,8 +24,11 @@ from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
_FIELDS: Final = TypeAdapter(dict[str, object])
_STRINGS: Final = TypeAdapter(tuple[str, ...])
_VALUES: Final = TypeAdapter(tuple[object, ...])
_ROWS: Final = TypeAdapter(tuple[dict[str, object], ...])
_TASK_TYPES: Final = frozenset(load_model_insight_tasks())
_MAX_INDEXED_TEXT_BYTES: Final = 2000
_MAX_COUNTER: Final = 2**40
@dataclass(frozen=True, slots=True)
@ -54,7 +57,8 @@ def _tags(row: Mapping[str, object]) -> tuple[str, ...]:
value: Final = row.get("request_tags")
if value is None:
return ()
return _STRINGS.validate_json(value) if isinstance(value, str) else _STRINGS.validate_python(value)
parsed: Final = _VALUES.validate_json(value) if isinstance(value, str) else _VALUES.validate_python(value)
return tuple(tag for tag in parsed if isinstance(tag, str))
def _date(row: Mapping[str, object]) -> str | None:
@ -191,13 +195,31 @@ def _window_contributions(row: Mapping[str, object], metadata: Mapping[str, obje
def project(row: Mapping[str, object]) -> tuple[Contribution, ...]:
metadata: Final = _fields(row.get("metadata"))
return (
contributions: Final = (
*_entity_contributions(row, metadata),
*_daily_contributions(row, metadata),
*_window_contributions(row, metadata),
*_model_contributions(row, metadata),
*_guardrail_contributions(row, metadata),
)
return tuple(_capped(contribution) for contribution in contributions if _indexable(contribution))
def _indexable(contribution: Contribution) -> bool:
texts: Final = (value for _, value in (*contribution.key, *contribution.values) if value is not None)
return all(len(text.encode("utf-8", "surrogatepass")) <= _MAX_INDEXED_TEXT_BYTES for text in texts)
def _capped(contribution: Contribution) -> Contribution:
"""Counters are summed into bigint columns in the charge's transaction, so one capped far below that range
can never overflow a day's total."""
return replace(
contribution,
increments=tuple(
(name, max(-_MAX_COUNTER, min(_MAX_COUNTER, value)) if isinstance(value, int) else value)
for name, value in contribution.increments
),
)
def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:

21
litellm/accounting/protocols.py
View file

@ -1,21 +0,0 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Protocol
from litellm.accounting.types import AggregateQuery, AggregateRow
class CostAggregator(Protocol):
async def apply(self, rows: tuple[Mapping[str, object], ...]) -> None:
"""Apply stored request contributions atomically with their delivery receipts."""
...
async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
"""Return disjoint [start, end) rows per group using existing daily projections.
Filters combine with AND, values within a filter with IN. DAY emits daily
buckets; None emits whole-interval totals. Reject unsupported metric/scope
combinations. Different groups may overlap, including non-additive tags.
"""
...

94
litellm/accounting/snapshot.py
View file

@ -2,12 +2,13 @@ from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure
from litellm.proxy._types import SpendLogsPayload
from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
from litellm.proxy.spend_tracking.savings import compute_savings_spend, marks_gateway_injection
from litellm.proxy.spend_tracking.savings import SavingsSpend, compute_savings_spend, marks_gateway_injection
if TYPE_CHECKING:
from litellm.router import Router
@ -66,7 +67,47 @@ def build_accounting_payload(
router: "Callable[[], Router | None]",
) -> Mapping[str, object]:
metadata: Final = _FIELDS.validate_json(payload["metadata"])
savings: Final = compute_savings_spend(
savings: Final = _savings_or_zero(payload, metadata, router)
accounting_metadata: Final = {
**{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS},
"accounting_entities": entities,
"accounting_savings": savings._asdict(),
"accounting_model_access_groups": model_access_groups,
"accounting_windows": windows,
}
return _FIELDS.validate_python(
safe_json_structure(
{
**{
key: value.isoformat() if isinstance(value, datetime) else value
for key, value in payload.items()
if key in _ACCOUNTING_FIELDS
},
"metadata": accounting_metadata,
},
non_finite_as_null=True,
)
)
def _savings_or_zero(
payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]"
) -> SavingsSpend:
"""Savings are analytics computed from caller-supplied counts, so one that cannot be computed is stored as zero
rather than losing the charge it rides with."""
try:
return _savings(payload, metadata, router)
except (ArithmeticError, ValueError, TypeError) as error:
verbose_proxy_logger.warning(
"Recording zero savings for request_id=%s, savings could not be computed: %s", payload["request_id"], error
)
return SavingsSpend(compression=0.0, prompt_caching=0.0)
def _savings(
payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]"
) -> SavingsSpend:
return compute_savings_spend(
model=payload["model"],
custom_llm_provider=payload["custom_llm_provider"],
compression_saved_tokens=extract_compression_saved_tokens(metadata),
@ -80,18 +121,41 @@ def build_accounting_payload(
recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")),
billed_at=payload["endTime"],
)
accounting_metadata: Final = {
**{key: value for key, value in metadata.items() if key in _METADATA_FIELDS},
"accounting_entities": entities,
"accounting_savings": savings._asdict(),
"accounting_model_access_groups": model_access_groups,
"accounting_windows": windows,
_GUARDRAIL_ENTRIES: Final = TypeAdapter(tuple[dict[str, object], ...])
_GUARDRAIL_METADATA_FIELDS: Final = frozenset({"guardrail_information", "standard_logging_guardrail_information"})
_GUARDRAIL_ACCOUNTING_FIELDS: Final = frozenset(
{
"guardrail_id",
"guardrail_name",
"guardrail_provider",
"guardrail_usage",
"guardrail_cost_by_unit",
"guardrail_cost_in_spend",
}
)
_GUARDRAIL_RESPONSE: Final = TypeAdapter(dict[str, object])
def _accounting_guardrail_entries(value: object) -> object:
try:
entries: Final = _GUARDRAIL_ENTRIES.validate_python((value,) if isinstance(value, dict) else value)
except ValidationError:
return None
return [_accounting_guardrail_entry(entry) for entry in entries]
def _accounting_guardrail_entry(entry: Mapping[str, object]) -> Mapping[str, object]:
kept: Final = {key: item for key, item in entry.items() if key in _GUARDRAIL_ACCOUNTING_FIELDS}
response: Final = entry.get("guardrail_response")
if not isinstance(response, dict) or "tokens_saved" not in response:
return kept
return {
**{
key: value.isoformat() if isinstance(value, datetime) else value
for key, value in payload.items()
if key in _ACCOUNTING_FIELDS
},
"metadata": _FIELDS.validate_python(safe_json_structure(accounting_metadata, non_finite_as_null=True)),
**kept,
"guardrail_response": {"tokens_saved": _GUARDRAIL_RESPONSE.validate_python(response)["tokens_saved"]},
}
def _accounting_metadata_field(key: str, value: object) -> object:
return _accounting_guardrail_entries(value) if key in _GUARDRAIL_METADATA_FIELDS else value

428
litellm/accounting/types.py
View file

@ -1,428 +0,0 @@
"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
from collections.abc import Mapping
from datetime import datetime, timezone
from decimal import Decimal
from enum import Enum
from typing import Annotated, Final, Literal, TypeAlias
from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
from typing_extensions import Self
from litellm.types.utils import CallTypes
_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
def _utc(value: datetime) -> datetime:
return value.astimezone(timezone.utc)
Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
class MetricName(str, Enum):
"""Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
REQUESTS = "requests"
SUCCESSFUL_REQUESTS = "successful_requests"
FAILED_REQUESTS = "failed_requests"
PROMPT_TOKENS = "prompt_tokens"
COMPLETION_TOKENS = "completion_tokens"
REASONING_TOKENS = "reasoning_tokens"
TOTAL_TOKENS = "total_tokens"
CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
SPEND = "spend"
class DimensionName(str, Enum):
KEY = "key"
USER = "user"
TEAM = "team"
ORGANIZATION = "organization"
END_USER = "end_user"
MODEL = "model"
MODEL_GROUP = "model_group"
DEPLOYMENT = "deployment"
PROVIDER = "provider"
TOOL = "tool"
AGENT = "agent"
PROJECT = "project"
MODEL_ACCESS_GROUP = "model_access_group"
API_BASE = "api_base"
SERVICE_TIER = "service_tier"
TAG = "tag"
class Outcome(str, Enum):
SUCCEEDED = "succeeded"
FAILED = "failed"
CANCELLED = "cancelled"
class Granularity(str, Enum):
DAY = "day"
class EventKind(str, Enum):
INBOUND_USAGE = "inbound_usage"
OUTBOUND_USAGE = "outbound_usage"
COST = "cost"
class DeliveryState(str, Enum):
PENDING = "pending"
APPLIED = "applied"
PARKED = "parked"
class DeliveryTarget(str, Enum):
"""One member per existing table the projector writes; each is applied and receipted independently."""
KEY_SPEND = "LiteLLM_VerificationToken"
USER_SPEND = "LiteLLM_UserTable"
TEAM_SPEND = "LiteLLM_TeamTable"
TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
END_USER_SPEND = "LiteLLM_EndUserTable"
AGENT_SPEND = "LiteLLM_AgentsTable"
TAG_SPEND = "LiteLLM_TagTable"
PROJECT_SPEND = "LiteLLM_ProjectTable"
MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
BUDGET_SPEND = "LiteLLM_BudgetTable"
BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
_SUBSET_PAIRS: Final = (
(MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
(MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
)
def _check_subsets(values: Mapping[str, Decimal]) -> None:
for part, whole in _SUBSET_PAIRS:
if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
raise ValueError(f"{part.value} cannot exceed {whole.value}")
class Metric(BaseModel):
model_config = _CONFIG
name: MetricName
value: Quantity
unit: Literal["count", "USD"] = "count"
@model_validator(mode="after")
def validate_unit(self) -> Self:
if (self.name is MetricName.SPEND) != (self.unit == "USD"):
raise ValueError("SPEND requires USD; usage metrics require count")
if self.unit == "count" and self.value != self.value.to_integral_value():
raise ValueError("Count metrics require integral values")
return self
class Dimension(BaseModel):
model_config = _CONFIG
name: DimensionName
value: Identifier
_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
if len(frozenset(values)) != len(values):
raise ValueError("Duplicate dimension values")
single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
if len(frozenset(single_names)) != len(single_names):
raise ValueError("Only tag and model access group dimensions may have multiple values")
return values
Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
if len(frozenset(value.name for value in values)) != len(values):
raise ValueError("Duplicate metric names")
return values
Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
if end < start or (end == start and not allow_empty):
raise ValueError("End must follow start")
class InboundUsageEvent(BaseModel):
model_config = _CONFIG
event_id: Identifier
request_id: Identifier
call_type: CallTypes
started_at: Instant
ended_at: Instant
outcome: Outcome
dimensions: Dimensions
metrics: Metrics
@model_validator(mode="after")
def validate_caller_activity(self) -> Self:
_interval(self.started_at, self.ended_at, allow_empty=True)
if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
raise ValueError("Inbound events own only caller request metrics")
if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
raise ValueError("Inbound events count exactly one caller request")
for metric in self.metrics:
if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
raise ValueError("Successful request count must match outcome")
if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
raise ValueError("Failed request count must match outcome")
return self
class OutboundUsageEvent(BaseModel):
model_config = _CONFIG
event_id: Identifier
attempt_id: Identifier
request_id: Identifier | None
retry_of_attempt_id: Identifier | None
call_type: CallTypes
started_at: Instant
ended_at: Instant
outcome: Outcome
dimensions: Dimensions
metrics: Metrics | None
@model_validator(mode="after")
def validate_attempt_usage(self) -> Self:
_interval(self.started_at, self.ended_at, allow_empty=True)
if self.retry_of_attempt_id == self.attempt_id:
raise ValueError("An attempt cannot retry itself")
if self.metrics is not None and any(
metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
):
raise ValueError("Outbound events own only attempt usage metrics")
if self.metrics is not None:
_check_subsets({metric.name.value: metric.value for metric in self.metrics})
return self
PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
class Rate(BaseModel):
model_config = _CONFIG
measure: Identifier
unit: PricingUnit
usd_per_unit: Quantity
class PricedQuantity(BaseModel):
"""A pricing input in its billing unit. Only seconds may be fractional."""
model_config = _CONFIG
measure: Identifier
unit: PricingUnit
value: Quantity
@model_validator(mode="after")
def validate_discrete_quantity(self) -> Self:
if self.unit != "second" and self.value != self.value.to_integral_value():
raise ValueError("Discrete priced quantities require integral values")
return self
class Multiplier(BaseModel):
"""A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
model_config = _CONFIG
name: Identifier
value: Identifier
factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
class NamedCost(BaseModel):
"""A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
model_config = _CONFIG
name: Identifier
amount: Quantity
class PricingSnapshot(BaseModel):
"""Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
Retries reuse the committed charge and snapshot.
Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
Component sums and float tolerances belong to reconciliation, not this structural contract.
"""
model_config = _CONFIG
schema_version: Literal[1] = 1
calculator_version: Identifier
model: Identifier
provider: Identifier | None
pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
priced_model_key: Identifier
multipliers: tuple[Multiplier, ...] = ()
rates: tuple[Rate, ...] = ()
usage: tuple[PricedQuantity, ...] = ()
input_cost: Quantity | None = None
output_cost: Quantity | None = None
cache_read_cost: Quantity | None = None
cache_creation_cost: Quantity | None = None
reasoning_cost: Quantity | None = None
tool_usage_cost: Quantity | None = None
guardrail_cost: Quantity | None = None
additional_costs: tuple[NamedCost, ...] = ()
original_cost: Quantity | None = None
discount_percent: Quantity | None = None
discount_amount: Quantity | None = None
margin_percent: Quantity | None = None
margin_fixed_amount: Quantity | None = None
margin_total_amount: Quantity | None = None
upstream_reported_cost: Quantity | None = None
total_cost: Quantity
@model_validator(mode="after")
def validate_snapshot(self) -> Self:
rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
if len(rate_keys) != len(self.rates):
raise ValueError("Duplicate rate measures and units")
if len(quantity_keys) != len(self.usage):
raise ValueError("Duplicate priced quantity measures and units")
if not rate_keys.issubset(quantity_keys):
raise ValueError("Every rate requires one priced quantity with the same measure and unit")
if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
raise ValueError("Duplicate additional cost names")
if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
raise ValueError("Duplicate multiplier names")
_check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
return self
class CostEvent(BaseModel):
model_config = _CONFIG
event_id: Identifier
usage_event_id: Identifier
occurred_at: Instant
dimensions: Dimensions
charges: Metrics
snapshot: PricingSnapshot
@model_validator(mode="after")
def validate_charge(self) -> Self:
if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
raise ValueError("A cost event requires one authoritative USD spend metric")
if self.event_id == self.usage_event_id:
raise ValueError("Cost and usage events require distinct identities")
if self.snapshot.total_cost != self.charges[0].value:
raise ValueError("Snapshot total must equal the authoritative spend")
return self
class Unpriced(BaseModel):
model_config = _CONFIG
usage_event_id: Identifier
reason: Identifier
class Measurement(BaseModel):
model_config = _CONFIG
source_event_id: Identifier
occurred_at: Instant
dimensions: Dimensions
metric: Metric
class DimensionFilter(BaseModel):
model_config = _CONFIG
name: DimensionName
values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
class AggregateQuery(BaseModel):
model_config = _CONFIG
metric: MetricName
filters: tuple[DimensionFilter, ...]
group_by: tuple[DimensionName, ...]
start: Instant
end: Instant
granularity: Granularity | None
@model_validator(mode="after")
def validate_query(self) -> Self:
_interval(self.start, self.end)
if len(frozenset(item.name for item in self.filters)) != len(self.filters):
raise ValueError("Use one IN filter per dimension; filters combine with AND")
if len(frozenset(self.group_by)) != len(self.group_by):
raise ValueError("Duplicate group-by dimensions")
if any(
(instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
for instant in (self.start, self.end)
):
raise ValueError("Initial reporting queries require UTC day boundaries")
return self
class AggregateRow(BaseModel):
model_config = _CONFIG
start: Instant
end: Instant
dimensions: Dimensions
value: Quantity
@model_validator(mode="after")
def validate_bounds(self) -> Self:
_interval(self.start, self.end)
return self
def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
"""Validate [start, end) buckets per group; overlapping tag groups are not additive."""
groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
for group in groups:
ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
raise ValueError("Aggregate buckets overlap within a dimension group")
return rows
class AdmissionDecision(BaseModel):
model_config = _CONFIG
allowed: bool
reason: Identifier | None = None

102
litellm/accounting/worker.py Normal file
View file

@ -0,0 +1,102 @@
import os
import time
from typing import (
TYPE_CHECKING,
Final,
Literal,
Protocol,
cast, # noqa: TID251 # generated Prisma transaction signatures cannot express the borrowed transaction protocol
)
from litellm._logging import verbose_proxy_logger
from litellm.accounting.application import apply_deliveries
from litellm.accounting.metrics import record_deliveries
from litellm.accounting.projection import Contribution
from litellm.repositories.accounting_delivery_repository import (
AccountingDeliveryRepository,
DeliveryProjection,
LedgerDatabase,
LedgerTransaction,
)
from litellm.types.integrations.slack_alerting import AlertType
if TYPE_CHECKING:
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.utils import PrismaClient, ProxyLogging
class _AccountingAlerts(Protocol):
async def alerting_handler(
self, message: str, level: Literal["Low", "Medium", "High"], alert_type: AlertType
) -> None: ...
_JOB: Final = "accounting_delivery"
async def flush_accounting_deliveries(
writer: "DBSpendUpdateWriter", prisma_client: "PrismaClient", proxy_logging_obj: "ProxyLogging"
) -> None:
acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False)
if acquired is False:
return
applied: tuple[Contribution, ...] = () # rebind-ok: keep committed contributions for cache invalidation
async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None:
nonlocal applied
applied = await apply_deliveries( # rebind-ok: invalidate caches only after the outer transaction commits
transaction, deliveries, prisma_client=prisma_client, writer=writer, proxy_logging_obj=proxy_logging_obj
)
try:
database: Final = cast( # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction
LedgerDatabase, prisma_client
)
repository: Final = AccountingDeliveryRepository(database)
started: Final = time.monotonic()
batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "10000"))
async def run_batch(*, retry_only: bool) -> bool:
result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
if result.applied:
await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
if result.parked:
verbose_proxy_logger.error(
"Accounting delivery parked %d requests after isolated failures", len(result.parked)
)
await _alert(
proxy_logging_obj,
message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.",
)
return bool(result.applied or result.failed)
await run_batch(retry_only=False)
retry_deadline: Final = time.monotonic() + 5
for _ in range(batch_size):
if time.monotonic() >= retry_deadline or not await run_batch(retry_only=True):
break
await repository.cleanup_applied(
retention_days=int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_RETENTION_DAYS", "1")),
batch_size=batch_size * 2,
)
metrics: Final = await repository.metrics()
record_deliveries(metrics, time.monotonic() - started)
if metrics.batch_count >= int(
os.environ.get("LITELLM_ACCOUNTING_BATCH_DELIVERY_ALERT_COUNT", "1000000")
) or metrics.table_bytes >= int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_ALERT_BYTES", "1073741824")):
await _alert(
proxy_logging_obj,
message=f"Accounting delivery storage requires review. Retained batches={metrics.batch_count}, table bytes={metrics.table_bytes}.",
)
if metrics.oldest_pending_seconds > 300:
await _alert(
proxy_logging_obj,
message=f"Accounting deliveries are delayed. Pending={metrics.pending_count}, oldest={metrics.oldest_pending_seconds:.0f}s.",
)
finally:
if acquired:
await writer.pod_lock_manager.release_lock(cronjob_id=_JOB)
async def _alert(alerts: _AccountingAlerts, message: str) -> None:
await alerts.alerting_handler(message=message, level="High", alert_type=AlertType.failed_tracking_spend)

3
litellm/constants.py
View file

@ -1673,6 +1673,9 @@ OPENAI_FINISH_REASONS: Final = [
]
HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute
RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = (
"When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text."
)
########################### Logging Callback Constants ###########################
AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07"

1
litellm/integrations/otel/model/spans.py
View file

@ -340,6 +340,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_AccessGroupTable",
"LiteLLM_AccountingDelivery",
"LiteLLM_AccountingEvent",
"LiteLLM_SpendLogDelivery",
"LiteLLM_ClaudeCodePluginTable",
"LiteLLM_ClaudeCodePluginUpload",
"LiteLLM_MemoryTable",

40
litellm/llms/anthropic/chat/handler.py
View file

@ -734,6 +734,7 @@ class ModelResponseIterator:
self._split_answer_blocks: tuple[str, ...] = ()
self._withheld_answer_close: str = ""
self._streamed_answer_keys: frozenset[str] | None = None
self._held_prose: str = ""
# For handling partial JSON chunks from fragmentation
# See: https://github.com/BerriAI/litellm/issues/17473
@ -826,8 +827,13 @@ class ModelResponseIterator:
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] = []
if "text" in content_block["delta"]:
text = content_block["delta"]["text"]
return text, tool_use, thinking_blocks, provider_specific_fields, reasoning_content
return (
self._hold_prose(content_block["delta"]["text"]),
tool_use,
thinking_blocks,
provider_specific_fields,
reasoning_content,
)
self.content_blocks.append(content_block)
if "partial_json" in content_block["delta"]:
# Only emit tool calls if we're in a tool_use or server_tool_use block
@ -1025,7 +1031,7 @@ class ModelResponseIterator:
if self.current_content_block_type in _THINKING_BLOCK_TYPES:
self.thinking_block_index += 1
if content_block_start["content_block"]["type"] == "text":
text = self._release_answer_close() + content_block_start["content_block"]["text"]
text = self._release_answer_close() + self._hold_prose(content_block_start["content_block"]["text"])
elif (
content_block_start["content_block"]["type"] == "tool_use"
or content_block_start["content_block"]["type"] == "server_tool_use"
@ -1161,7 +1167,7 @@ class ModelResponseIterator:
elif type_chunk == "message_delta":
finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(stop_reason_fields)
text = self._release_answer_close() + self._drain_split_answer_blocks()
text = self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose()
if container:
provider_specific_fields["container"] = container
# Store container_id and re-emit code_interpreter_results
@ -1211,7 +1217,9 @@ class ModelResponseIterator:
status_code=500, # it looks like Anthropic API does not return a status code in the chunk error - default to 500
)
text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
text, tool_use = self._handle_json_mode_chunk(
text=text + self._settle_held_prose(tool_use), tool_use=tool_use
)
refusal_fields: Final = refusal_params(provider_specific_fields)
returned_chunk: Final = ModelResponseStream(
@ -1309,6 +1317,24 @@ class ModelResponseIterator:
self._response_format_verdict = _envelope_verdict(buffered, schema, RESPONSE_FORMAT_TOOL_NAME)
return self._withhold_answer_close(buffered) if self._response_format_verdict == "pass_through" else ""
def _hold_prose(self, text: str) -> str:
if not self.json_mode or self.response_format_schema is None:
return text
self._held_prose += text
return ""
def _settle_held_prose(self, tool_use: ChatCompletionToolCallChunk | None) -> str:
if tool_use is None or tool_use.get("id") is None or self.current_content_block_type != "tool_use":
return ""
held: Final = self._held_prose
self._held_prose = ""
return "" if tool_use.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME else held
def _release_held_prose(self) -> str:
held: Final = self._held_prose
self._held_prose = ""
return "" if self.converted_response_format_tool else held
def _withhold_answer_close(self, text: str) -> str:
pending: Final = self._withheld_answer_close + text
body: Final = pending.rstrip()
@ -1362,7 +1388,9 @@ class ModelResponseIterator:
def cut_stream_chunk(self) -> ModelResponseStream | None:
unsent: Final = self._flush_response_format_buffer() if self._response_format_buffer.getvalue() else ""
text: Final = unsent + self._release_answer_close() + self._drain_split_answer_blocks()
text: Final = (
unsent + self._release_answer_close() + self._drain_split_answer_blocks() + self._release_held_prose()
)
if not text:
return None
return ModelResponseStream(

31
litellm/llms/anthropic/chat/transformation.py
View file

@ -26,6 +26,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
RESPONSE_FORMAT_TOOL_NAME,
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION,
)
from litellm.litellm_core_utils.asyncify import is_event_loop_running
from litellm.litellm_core_utils.core_helpers import (
@ -1786,7 +1787,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
open_object_takes_forced_json_tool: Final = (
open_object_takes_json_tool: Final = (
json_schema is not None
and schema_has_open_object(json_schema)
and not self.is_thinking_requested(request_params)
@ -1794,7 +1795,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
and not request_params.get("tools")
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
)
return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool
return self.uses_native_structured_output(model) and not open_object_takes_json_tool
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema)))
@ -1809,7 +1810,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema))
def map_response_format_to_anthropic_tool(
self, value: dict | None, optional_params: dict, is_thinking_enabled: bool
self, value: dict | None, optional_params: dict, tool_is_forced: bool
) -> AnthropicMessagesTool | None:
ignore_response_format_types: Final = ["text"]
if value is None or value["type"] in ignore_response_format_types: # value is a no-op
@ -1827,6 +1828,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_tool: Final = self._create_json_tool_call_for_response_format(
json_schema=json_schema,
description=None if tool_is_forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION,
)
return _tool
@ -1929,7 +1931,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
drop_params=drop_params,
runs_hosted_mcp=self.runs_hosted_mcp,
)
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params=request_params)
response_format_tool_is_forced: Final = (
"response_format" in request_params
and AnthropicModelInfo.response_format_tool_can_be_forced(
model, self._resolved_provider, REQUEST_PARAMS.validate_python(request_params)
)
)
# NB: ``map_openai_params`` deliberately does NOT sanitize tool names
# here. Names are the *original* OpenAI names at this stage, and must
@ -1989,10 +1996,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if _output_format is not None:
optional_params["output_format"] = _output_format
else:
_tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled)
_tool = self.map_response_format_to_anthropic_tool(
value, optional_params, response_format_tool_is_forced
)
if _tool is None:
continue
if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model):
if response_format_tool_is_forced:
_tool_choice = {
"name": RESPONSE_FORMAT_TOOL_NAME,
"type": "tool",
@ -2121,12 +2130,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def _create_json_tool_call_for_response_format(
self,
json_schema: dict | None = None,
description: str | None = None,
) -> AnthropicMessagesTool:
"""
Handles creating a tool call for getting responses in JSON format.
Args:
json_schema (Optional[dict]): The JSON schema the response should be in
description (Optional[str]): The tool description, left off the tool when None
Returns:
AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format
@ -2143,11 +2154,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else:
_input_schema.update(cast(AnthropicInputSchema, json_schema))
_tool: Final = AnthropicMessagesTool(
name=RESPONSE_FORMAT_TOOL_NAME,
input_schema=_input_schema,
if description is None:
return AnthropicMessagesTool(name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema)
return AnthropicMessagesTool(
name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema, description=description
)
return _tool
def should_strip_billing_metadata(self) -> bool:
"""

28
litellm/llms/anthropic/common_utils.py
View file

@ -1141,6 +1141,34 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"""
return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
@staticmethod
def thinks_by_default(model: str, custom_llm_provider: str) -> bool:
return AnthropicModelInfo._is_always_on_thinking_model(
model, custom_llm_provider
) or AnthropicModelInfo._supports_model_capability(model, "thinking_on_by_default", custom_llm_provider)
@staticmethod
def will_think(model: str, custom_llm_provider: str, request_params: Mapping[str, object]) -> bool:
if AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider):
return True
match request_params:
case {"thinking": True | {"type": "enabled" | "adaptive"}}:
return True
case {"reasoning_effort": effort} if effort is not None and effort != "none":
return True
case {"thinking": {"type": "disabled"}} | {"reasoning_effort": "none"}:
return False
case _:
return AnthropicModelInfo.thinks_by_default(model, custom_llm_provider)
@staticmethod
def response_format_tool_can_be_forced(
model: str, custom_llm_provider: str, request_params: Mapping[str, object]
) -> bool:
return not AnthropicModelInfo.forced_tool_use_unsupported(model) and not AnthropicModelInfo.will_think(
model, custom_llm_provider, request_params
)
@staticmethod
def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` is an adaptive-thinking model that still accepts legacy

88
litellm/llms/azure/responses/transformation.py
View file

@ -3,11 +3,10 @@ from collections.abc import Callable, Mapping
from copy import deepcopy
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from urllib.parse import urlsplit
import httpx
from openai.types.responses import ResponseReasoningItem
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
@ -16,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cach
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
@ -72,6 +72,21 @@ def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[st
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
_AZURE_REJECTED_INPUT_FIELDS: Final = MappingProxyType(
{"message": frozenset({"status"}), "reasoning": frozenset({"status"})}
)
_AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS: Final = MappingProxyType(
{**_AZURE_REJECTED_INPUT_FIELDS, "reasoning": frozenset({"status", "content"})}
)
def _without_fields_azure_rejects(item: object, rejected_fields: Mapping[str, frozenset[str]]) -> object:
if not is_str_keyed_mapping(item):
return item
rejected: Final = rejected_fields.get(str(item.get("type")), frozenset())
return {key: value for key, value in item.items() if key not in rejected} if rejected else item
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Parameters not supported by Azure Responses API
AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@ -122,68 +137,17 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def get_stripped_model_name(self, model: str) -> str:
return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]:
"""
Handle reasoning items to filter out the status field.
Issue: https://github.com/BerriAI/litellm/issues/13484
Azure OpenAI API does not accept 'status' field in reasoning input items.
"""
if item.get("type") == "reasoning":
try:
# Ensure required fields are present for ResponseReasoningItem
item_data: Final = dict(item)
if "summary" not in item_data:
item_data["summary"] = (
item_data.get("reasoning_content", "")[:100] + "..."
if len(item_data.get("reasoning_content", "")) > 100
else item_data.get("reasoning_content", "")
)
# Create ResponseReasoningItem object from the item data
reasoning_item: Final = ResponseReasoningItem.model_validate(item_data)
# Convert back to dict with exclude_none=True to exclude None fields
dict_reasoning_item: Final = reasoning_item.model_dump(exclude_none=True)
dict_reasoning_item.pop("status", None)
return dict_reasoning_item
except Exception as e:
verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e)
# Fallback: manually filter out known None fields
filtered_item: Final = {
k: v
for k, v in item.items()
if v is not None or k not in {"status", "content", "encrypted_content"}
}
return filtered_item
return item
def _validate_input_param(
self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
self, input: str | ResponseInputParam, *, drop_params: bool = False
) -> str | ResponseInputParam:
"""
Override parent method to also filter out 'status' field from message items.
Azure OpenAI API does not accept 'status' field in input messages.
"""
from typing import cast
# First call parent's validation
validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by)
# Then filter out status from message items
if isinstance(validated_input, list):
filtered_input: Final[list[object]] = []
for item in validated_input:
if isinstance(item, dict) and item.get("type") == "message":
# Filter out status field from message items
filtered_item = {k: v for k, v in item.items() if k != "status"}
filtered_input.append(filtered_item)
else:
filtered_input.append(item)
return cast(ResponseInputParam, filtered_input)
return validated_input
validated_input: Final = super()._validate_input_param(input, drop_params=drop_params)
if not isinstance(validated_input, list):
return validated_input
rejected_fields: Final = (
_AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS if drop_params else _AZURE_REJECTED_INPUT_FIELDS
)
sanitized_items: Final = [_without_fields_azure_rejects(item, rejected_fields) for item in validated_input]
return cast("ResponseInputParam", sanitized_items) # cast-ok: items keep their shape minus rejected fields
def transform_responses_api_request(
self,

59
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -22,6 +22,7 @@ from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_hea
from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
RESPONSE_FORMAT_TOOL_NAME,
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION,
)
from litellm.integrations.anthropic_cache_control_hook import (
MAX_CACHE_CONTROL_BLOCKS,
@ -950,19 +951,24 @@ class AmazonConverseConfig(BaseConfig):
def _supports_native_structured_outputs(
model: str,
custom_llm_provider: str | None,
will_think: bool,
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> bool:
"""Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when the
request asks for thinking, which the json tool path would drop. Without thinking, a model that cannot
force the json tool gets it unless that key is false.
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude
will think, which the json tool path would drop. When the request does not ask for thinking, a model
that cannot force the json tool gets it unless that key is false.
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
"""
return bedrock_chat_uses_native_structured_output(
model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info
model,
will_think=will_think,
thinking_requested=thinking_requested,
custom_llm_provider=custom_llm_provider or "bedrock_converse",
model_info=model_info,
)
@staticmethod
@ -1005,11 +1011,13 @@ class AmazonConverseConfig(BaseConfig):
self,
model: str,
json_schema: Mapping[str, object],
*,
will_think: bool,
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if not self._supports_native_structured_outputs(
model, self.custom_llm_provider, thinking_requested, model_info
model, self.custom_llm_provider, will_think, thinking_requested, model_info
):
return None
if BedrockModelInfo.get_base_model(model).startswith("anthropic"):
@ -1095,7 +1103,6 @@ class AmazonConverseConfig(BaseConfig):
request_params: Final = tool_params_without_hosted_mcp(
non_default_params, model=model, route="bedrock", drop_params=drop_params
)
is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
base_model: Final = BedrockModelInfo.get_base_model(model)
drop_thinking_param: Final = self._is_deepseek_model(model=model, base_model=base_model)
drop_reasoning_effort_param: Final = self._model_rejects_reasoning_effort_param(
@ -1109,7 +1116,6 @@ class AmazonConverseConfig(BaseConfig):
model=model,
optional_params=optional_params,
non_default_params=request_params,
is_thinking_enabled=is_thinking_enabled,
model_info=model_info,
)
if param == "max_tokens" or param == "max_completion_tokens":
@ -1314,7 +1320,6 @@ class AmazonConverseConfig(BaseConfig):
model: str,
optional_params: dict,
non_default_params: dict,
is_thinking_enabled: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
"""
@ -1339,9 +1344,16 @@ class AmazonConverseConfig(BaseConfig):
if "type" in value and value["type"] == "text":
return optional_params
thinking_requested: Final = self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params))
request_params: Final = REQUEST_PARAMS.validate_python(non_default_params)
thinking_requested: Final = self.is_thinking_requested(request_params)
native_schema: Final = (
self._native_output_schema(model, json_schema, thinking_requested, model_info)
self._native_output_schema(
model,
json_schema,
will_think=AnthropicModelInfo.will_think(model, "bedrock", request_params),
thinking_requested=thinking_requested,
model_info=model_info,
)
if json_schema is not None
else None
)
@ -1359,17 +1371,24 @@ class AmazonConverseConfig(BaseConfig):
elif json_schema is not None:
# Fallback: translate to a synthetic tool call
# https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
tool_is_forced: Final = litellm.utils.supports_tool_choice(
model=model, custom_llm_provider=self.custom_llm_provider
) and AnthropicModelInfo.response_format_tool_can_be_forced(model, "bedrock", request_params)
schema_description: Final = description if isinstance(description, str) else None
instructs_unforced_tool: Final = not tool_is_forced and BedrockModelInfo.get_base_model(model).startswith(
"anthropic"
)
_tool: Final = self._create_json_tool_call_for_response_format(
json_schema=json_schema,
description=description,
description=(
" ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, schema_description)))
if instructs_unforced_tool
else schema_description
),
)
optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool])
if (
litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
and not is_thinking_enabled
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
):
if tool_is_forced:
optional_params["tool_choice"] = ToolChoiceValuesBlock(
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
)
@ -1380,8 +1399,8 @@ class AmazonConverseConfig(BaseConfig):
# returns {} instead of the requested JSON. The model already
# returns JSON when the prompt asks for it.
# A request that goes native only because the model can't force the json tool keeps the json tool's faked
# stream, so it still needs only bedrock:InvokeModel.
# A request that goes native without asking for thinking keeps the json tool's faked stream, so it still
# needs only bedrock:InvokeModel, unless the model takes native output on every request.
streams_natively: Final = native_schema is not None and bedrock_chat_opts_into_native_structured_output(
model, thinking_requested, self.custom_llm_provider or "bedrock_converse", model_info
)
@ -2219,8 +2238,8 @@ class AmazonConverseConfig(BaseConfig):
}
if additional_request_params:
data["additionalModelRequestFields"] = additional_request_params
if "thinking" in additional_request_params:
data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",)
if "thinking" in additional_request_params or AnthropicModelInfo.thinks_by_default(model, "bedrock"):
data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",)
if system_content_blocks:
data["system"] = system_content_blocks

17
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py
View file

@ -142,14 +142,19 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
return self.chat_uses_native_structured_output(
model, self.is_thinking_requested(request_params), self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
return self.chat_uses_native_structured_output(model, request_params, self._deployment_model_info) and (
json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None
)
def chat_uses_native_structured_output(
self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None
) -> bool:
return bedrock_chat_uses_native_structured_output(model, thinking_requested, model_info=model_info)
return bedrock_chat_uses_native_structured_output(
model,
will_think=self.will_think(model, "bedrock", request_params),
thinking_requested=self.is_thinking_requested(request_params),
model_info=model_info,
)
def uses_opt_in_native_output(
self, model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
@ -359,7 +364,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
request_body=anthropic_request,
model_info=model_info,
chat_uses_native_output=lambda: self.chat_uses_native_structured_output(
model, self.is_thinking_requested(_NON_DEFAULT_PARAMS.validate_python(filtered_params)), model_info
model, _NON_DEFAULT_PARAMS.validate_python(filtered_params), model_info
),
)
strip_unsupported_bedrock_invoke_output_config_keys(

6
litellm/llms/bedrock/chat/mantle/transformation.py
View file

@ -55,9 +55,11 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
)
def chat_uses_native_structured_output(
self, model: str, thinking_requested: bool, model_info: Mapping[str, object] | None
self, model: str, request_params: Mapping[str, object], model_info: Mapping[str, object] | None
) -> bool:
return bedrock_chat_opts_into_native_structured_output(model, thinking_requested, model_info=model_info)
return bedrock_chat_opts_into_native_structured_output(
model, self.will_think(model, "bedrock", request_params), model_info=model_info
)
def _default_max_tokens_models(self, model: str, litellm_params: Mapping[str, object]) -> tuple[str, ...]:
return (f"bedrock_mantle/{model}", *super()._default_max_tokens_models(model, litellm_params))

8
litellm/llms/bedrock/common_utils.py
View file

@ -357,13 +357,15 @@ def bedrock_chat_opts_into_native_structured_output(
def bedrock_chat_uses_native_structured_output(
model: str,
*,
will_think: bool,
thinking_requested: bool,
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return bedrock_chat_opts_into_native_structured_output(
model, thinking_requested, custom_llm_provider, model_info
) or (not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info))
return bedrock_chat_opts_into_native_structured_output(model, will_think, custom_llm_provider, model_info) or (
not thinking_requested and _bedrock_native_output_replaces_unforceable_tool(model, model_info)
)
def bedrock_native_output_is_deployment_opt_in(

13
litellm/llms/databricks/chat/transformation.py
View file

@ -31,6 +31,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.base_utils import (
type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict
)
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.databricks import (
AllDatabricksContentValues,
@ -541,12 +542,12 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
model: str,
value: dict | None,
optional_params: dict,
is_thinking_enabled: bool,
tool_is_forced: bool,
) -> DatabricksTool | None:
if value is None:
return None
tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled)
tool: Final = self.map_response_format_to_anthropic_tool(value, optional_params, tool_is_forced)
databricks_tool: Final = self.convert_anthropic_tool_to_databricks_tool(tool)
return databricks_tool
@ -579,7 +580,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
if "claude" in model
else non_default_params
)
is_thinking_enabled: Final = self.is_thinking_enabled(request_params)
mapped_params: Final = super().map_openai_params(request_params, optional_params, model, drop_params)
if "claude" in model:
AnthropicConfig.translate_legacy_thinking_for_adaptive_model(
@ -596,17 +596,20 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
mapped_params.pop("max_completion_tokens", None)
if "response_format" in request_params and "claude" in model:
tool_is_forced: Final = AnthropicConfig.response_format_tool_can_be_forced(
model, "databricks", REQUEST_PARAMS.validate_python(request_params)
)
_tool: Final = self.map_response_format_to_databricks_tool(
model,
request_params["response_format"],
mapped_params,
is_thinking_enabled,
tool_is_forced,
)
if _tool is not None:
self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool])
optional_params["json_mode"] = True
if not is_thinking_enabled:
if tool_is_forced:
_tool_choice: Final = ChatCompletionToolChoiceObjectParam(
type="function",
function=ChatCompletionToolChoiceFunctionParam(name=RESPONSE_FORMAT_TOOL_NAME),

328
litellm/llms/nvidia_riva/audio_transcription/handler.py
View file

@ -13,24 +13,26 @@ The handler is intentionally a thin orchestration layer:
2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs
from the structured dict produced by
:class:`NvidiaRivaAudioTranscriptionConfig`.
3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS)
vs self-hosted (any host:port, optional TLS) modes.
4. Stream the audio through Riva's ``streaming_response_generator`` and
aggregate ``is_final`` results into a single transcript, or, when
``riva_offline`` is set, send the whole buffer through the unary
``offline_recognize`` RPC for NIMs that only ship offline engines.
3. Open a gRPC channel honoring NVCF (TLS by default) vs self-hosted (any
host:port, optional TLS) modes, and attach the ``authorization`` and NVCF
``function-id`` metadata to each call exactly once.
4. Stream the audio through Riva's ``StreamingRecognize`` RPC and aggregate
``is_final`` results into a single transcript, or, when ``riva_offline``
is set, send the whole buffer through the unary ``Recognize`` RPC for NIMs
that only ship offline engines.
5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed
on ``_hidden_params`` so cost calculation works.
``riva-client`` is imported lazily so ``litellm`` core remains usable
without the optional STT extras installed.
``grpcio`` and ``protobuf`` are imported lazily so ``litellm`` core remains
usable without the optional STT extras installed.
"""
from __future__ import annotations
import asyncio
import inspect
from collections.abc import Callable, Iterable, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol
from collections.abc import Iterable, Sequence
from contextlib import closing
from typing import TYPE_CHECKING, Final
import httpx
@ -64,89 +66,19 @@ if TYPE_CHECKING:
Logging as LiteLLMLoggingObj,
)
# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches
# NVIDIA's recommended chunk size for streaming ASR — small enough for
# responsive endpointing, large enough to keep per-RPC overhead low.
_DEFAULT_CHUNK_SAMPLES: Final = 1600
_DEFAULT_CHUNK_BYTES: Final = _DEFAULT_CHUNK_SAMPLES * 2 # int16 = 2 bytes/sample
_RIVA_INSTALL_HINT = "NVIDIA Riva client is not installed. Install with `pip install 'litellm[stt-nvidia-riva]'`."
class _RivaAuth(Protocol):
"""Opaque ``riva.client.Auth`` handle."""
class _RecognizeFuture(Protocol):
def result(self, timeout: float | None = None) -> object: ...
def done(self) -> bool: ...
def cancel(self) -> bool: ...
class _AsrService(Protocol):
@property
def streaming_response_generator(self) -> Callable[..., Iterable[object]]: ...
def offline_recognize(self, audio_bytes: bytes, config: object, future: bool) -> _RecognizeFuture: ...
class _EndpointingConfig(Protocol):
"""Opaque ``EndpointingConfig`` protobuf message."""
class _EndpointingConfigField(Protocol):
CopyFrom: Callable[[_EndpointingConfig], None]
class _RecognitionConfig(Protocol):
@property
def endpointing_config(self) -> _EndpointingConfigField: ...
class _StreamingRecognitionConfig(Protocol):
"""Opaque ``StreamingRecognitionConfig`` protobuf message."""
class _AudioEncoding(Protocol):
@property
def LINEAR_PCM(self) -> object: ...
class _RivaClientModule(Protocol):
"""The ``riva.client`` entry points this handler calls."""
@property
def Auth(self) -> Callable[..., _RivaAuth]: ...
@property
def ASRService(self) -> Callable[[_RivaAuth], _AsrService]: ...
class _RivaAsrModule(Protocol):
"""The protobuf constructors this handler calls, from whichever module exposes them."""
@property
def AudioEncoding(self) -> _AudioEncoding: ...
@property
def RecognitionConfig(self) -> Callable[..., _RecognitionConfig]: ...
@property
def StreamingRecognitionConfig(self) -> Callable[..., _StreamingRecognitionConfig]: ...
@property
def EndpointingConfig(self) -> Callable[..., _EndpointingConfig]: ...
def _auth_factory(riva_module: _RivaClientModule) -> Callable[..., _RivaAuth]:
return riva_module.Auth
def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding:
return riva_asr_module.AudioEncoding
def _use_ssl(optional_params: dict) -> bool:
"""
NVCF endpoints are TLS-only, so ``nvcf_function_id`` defaults ``use_ssl``
to True. Self-hosted Riva defaults to plaintext but honors an explicit
``use_ssl``, since TLS-terminating ingresses in front of it are common.
"""
use_ssl_override: Final = optional_params.get("use_ssl")
if use_ssl_override is not None:
return bool(use_ssl_override)
return bool(optional_params.get("nvcf_function_id"))
def _parse_riva_offline(value: object, drop_params: bool) -> bool:
@ -192,7 +124,7 @@ class NvidiaRivaAudioTranscription:
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float | str | httpx.Timeout | None,
logging_obj: "LiteLLMLoggingObj",
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
atranscription: bool = False,
@ -237,13 +169,11 @@ class NvidiaRivaAudioTranscription:
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float | str | httpx.Timeout | None,
logging_obj: "LiteLLMLoggingObj",
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
provider_config: NvidiaRivaAudioTranscriptionConfig | None = None,
) -> TranscriptionResponse:
# ``riva-client`` exposes a sync streaming generator, so we offload
# the blocking call to a worker thread to keep the event loop free.
return await asyncio.to_thread(
self._run_sync,
model=model,
@ -267,7 +197,7 @@ class NvidiaRivaAudioTranscription:
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float | str | httpx.Timeout | None,
logging_obj: "LiteLLMLoggingObj",
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
provider_config: NvidiaRivaAudioTranscriptionConfig,
@ -317,18 +247,12 @@ class NvidiaRivaAudioTranscription:
) or litellm.drop_params is True
riva_offline: Final = _parse_riva_offline(optional_params.get("riva_offline"), drop_params=drop_params)
riva_module, riva_asr_module = _import_riva()
auth_obj: Final = self._construct_auth(
riva_module=riva_module,
api_base=api_base,
api_key=api_key,
optional_params=optional_params,
)
try:
from litellm.llms.nvidia_riva.audio_transcription import riva_grpc
except ImportError as e:
raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
recognition_config: Final = self._build_recognition_config_proto(
riva_asr_module=riva_asr_module,
recognition_config_dict=recognition_config_dict,
)
recognition_config: Final = riva_grpc.recognition_config(recognition_config_dict)
logging_obj.pre_call(
input=None,
api_key=api_key,
@ -344,26 +268,18 @@ class NvidiaRivaAudioTranscription:
},
)
metadata: Final = riva_grpc.call_metadata(api_key, optional_params.get("nvcf_function_id"))
try:
asr_service: Final[_AsrService] = riva_module.ASRService(auth_obj)
responses: Final = (
self._offline_responses(
asr_service=asr_service,
pcm_bytes=resampled.pcm_bytes,
recognition_config=recognition_config,
timeout=None if timeout is None else _deadline_seconds(timeout, drop_params),
deadline: Final = None if timeout is None else _deadline_seconds(timeout, drop_params)
with closing(riva_grpc.open_channel(api_base, _use_ssl(optional_params))) as channel:
responses: Final = (
riva_grpc.recognize(channel, recognition_config, resampled.pcm_bytes, metadata, deadline)
if riva_offline
else riva_grpc.streaming_recognize(
channel, recognition_config, resampled.pcm_bytes, metadata, deadline
)
)
if riva_offline
else self._streaming_responses(
asr_service=asr_service,
riva_asr_module=riva_asr_module,
pcm_bytes=resampled.pcm_bytes,
recognition_config=recognition_config,
timeout=timeout,
drop_params=drop_params,
)
)
final_results: Final = self._collect_final_results(responses, require_final=not riva_offline)
final_results: Final = self._collect_final_results(responses, require_final=not riva_offline)
except NvidiaRivaException:
raise
except Exception as e:
@ -400,142 +316,6 @@ class NvidiaRivaAudioTranscription:
return final_response
def _construct_auth(
self,
riva_module: _RivaClientModule,
api_base: str,
api_key: str | None,
optional_params: dict,
) -> _RivaAuth:
"""
Build a ``riva.client.Auth`` object.
- When ``nvcf_function_id`` is provided we attach the NVCF
``function-id`` and bearer ``authorization`` metadata, and default
``use_ssl`` to True (NVCF endpoints are TLS-only).
- Otherwise (self-hosted) we default ``use_ssl`` to False but still
honor an explicit override — self-hosted Riva behind an ingress
with TLS termination is a real deployment topology.
"""
nvcf_function_id: Final = optional_params.get("nvcf_function_id")
use_ssl_override: Final = optional_params.get("use_ssl")
use_ssl: Final = bool(use_ssl_override) if use_ssl_override is not None else bool(nvcf_function_id)
metadata: Final[list[tuple[str, str]]] = []
if nvcf_function_id:
metadata.append(("function-id", str(nvcf_function_id)))
if api_key:
metadata.append(("authorization", f"Bearer {api_key}"))
try:
return _auth_factory(riva_module)(uri=api_base, use_ssl=use_ssl, metadata_args=metadata)
except TypeError:
# Older riva-client signatures used positional-only args.
return _auth_factory(riva_module)(None, use_ssl, api_base, metadata)
def _build_recognition_config_proto(
self, riva_asr_module: _RivaAsrModule, recognition_config_dict: dict[str, Any]
) -> _RecognitionConfig:
encoding_name: Final = (recognition_config_dict.get("encoding") or "LINEAR_PCM").upper()
encoding_enum: Final[object] = getattr(
_audio_encoding(riva_asr_module),
encoding_name,
_audio_encoding(riva_asr_module).LINEAR_PCM,
)
config: Final[_RecognitionConfig] = riva_asr_module.RecognitionConfig(
encoding=encoding_enum,
sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]),
language_code=recognition_config_dict["language_code"],
audio_channel_count=int(recognition_config_dict["audio_channel_count"]),
enable_automatic_punctuation=bool(recognition_config_dict.get("enable_automatic_punctuation", True)),
enable_word_time_offsets=bool(recognition_config_dict.get("enable_word_time_offsets", False)),
max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)),
model=recognition_config_dict.get("model", "") or "",
verbatim_transcripts=bool(recognition_config_dict.get("verbatim_transcripts", False)),
profanity_filter=bool(recognition_config_dict.get("profanity_filter", False)),
)
endpointing: Final = recognition_config_dict.get("endpointing_config")
if isinstance(endpointing, dict) and endpointing:
try:
ep: Final[_EndpointingConfig] = riva_asr_module.EndpointingConfig(**endpointing)
config.endpointing_config.CopyFrom(ep)
except Exception:
# If the user supplied an unknown EndpointingConfig field
# (older Riva server), fall back to Riva's defaults rather
# than failing the whole request.
pass
return config
def _streaming_responses(
self,
asr_service: _AsrService,
riva_asr_module: _RivaAsrModule,
pcm_bytes: bytes,
recognition_config: _RecognitionConfig,
timeout: float | str | httpx.Timeout | None,
drop_params: bool,
) -> Iterable[object]:
streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig(
config=recognition_config, interim_results=False
)
return asr_service.streaming_response_generator(
audio_chunks=self._iter_audio_chunks(pcm_bytes),
streaming_config=streaming_config,
**self._timeout_kwargs(asr_service.streaming_response_generator, timeout, drop_params),
)
def _offline_responses(
self,
asr_service: _AsrService,
pcm_bytes: bytes,
recognition_config: _RecognitionConfig,
timeout: float | None,
) -> Iterable[object]:
pending: Final = asr_service.offline_recognize(audio_bytes=pcm_bytes, config=recognition_config, future=True)
try:
return (pending.result(timeout=timeout),)
except Exception as e:
if pending.done():
raise
pending.cancel()
raise NvidiaRivaException(
status_code=408,
message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds",
) from e
@classmethod
def _timeout_kwargs(
cls, rpc: Callable[..., object], timeout: float | str | httpx.Timeout | None, drop_params: bool
) -> Mapping[str, float]:
# Forward the deadline so the call cannot block forever if the
# server stalls. Older riva-client versions do not accept a
# ``timeout`` kwarg, so pass it only when supported.
if timeout is None or not cls._supports_timeout_kwarg(rpc):
return MappingProxyType({})
return MappingProxyType({"timeout": _deadline_seconds(timeout, drop_params)})
@staticmethod
def _supports_timeout_kwarg(callable_obj: Callable[..., object]) -> bool:
try:
sig: Final = inspect.signature(callable_obj)
except (TypeError, ValueError):
return False
params: Final = sig.parameters
if "timeout" in params:
return True
return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
@staticmethod
def _iter_audio_chunks(pcm_bytes: bytes):
for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES):
chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES]
if not chunk:
continue
yield chunk
@staticmethod
def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> Sequence[RivaFinalResult]:
"""
@ -570,27 +350,3 @@ class NvidiaRivaAudioTranscription:
}
)
return final_results
def _import_riva() -> tuple[_RivaClientModule, _RivaAsrModule]:
"""
Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``.
We try the SDK first (preferred) and fall back to importing the proto
module separately when the SDK packaging changes between versions.
"""
try:
import riva.client as riva_client
except ImportError as e:
raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
riva_asr_module = riva_client
if not hasattr(riva_asr_module, "RecognitionConfig"):
try:
from riva.client.proto import riva_asr_pb2
riva_asr_module = riva_asr_pb2
except ImportError as e:
raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
return riva_client, riva_asr_module

199
litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py Normal file
View file

@ -0,0 +1,199 @@
"""
The part of NVIDIA Riva's speech recognition gRPC schema that LiteLLM sends and reads.
Names and numbers match ``riva/proto/riva_asr.proto`` and ``riva/proto/riva_audio.proto`` in
https://github.com/nvidia-riva/common. Fields LiteLLM never sets or reads are left out, and the
wire format skips them. The messages live in their own descriptor pool so they never collide
with nvidia-riva-client's copies when both are imported in one process.
"""
from dataclasses import dataclass
from enum import Enum, IntEnum
from typing import Final
from google.protobuf import descriptor_pb2, descriptor_pool, message_factory
from google.protobuf.message import Message
_PACKAGE: Final = "nvidia.riva.asr"
_SERVICE: Final = f"/{_PACKAGE}.RivaSpeechRecognition"
RECOGNIZE_METHOD: Final = f"{_SERVICE}/Recognize"
STREAMING_RECOGNIZE_METHOD: Final = f"{_SERVICE}/StreamingRecognize"
class _Kind(Enum):
FLOAT = "TYPE_FLOAT"
INT32 = "TYPE_INT32"
BOOL = "TYPE_BOOL"
STRING = "TYPE_STRING"
MESSAGE = "TYPE_MESSAGE"
BYTES = "TYPE_BYTES"
ENUM = "TYPE_ENUM"
class AudioEncoding(IntEnum):
ENCODING_UNSPECIFIED = 0
LINEAR_PCM = 1
LINEAR_PCM: Final = AudioEncoding.LINEAR_PCM.value
@dataclass(frozen=True, slots=True)
class _Field:
name: str
number: int
kind: _Kind
type_name: str = ""
repeated: bool = False
optional: bool = False
oneof: str = ""
_MESSAGES: Final[tuple[tuple[str, tuple[_Field, ...]], ...]] = (
(
"RecognitionConfig",
(
_Field("encoding", 1, _Kind.ENUM, "AudioEncoding"),
_Field("sample_rate_hertz", 2, _Kind.INT32),
_Field("language_code", 3, _Kind.STRING),
_Field("max_alternatives", 4, _Kind.INT32),
_Field("profanity_filter", 5, _Kind.BOOL),
_Field("audio_channel_count", 7, _Kind.INT32),
_Field("enable_word_time_offsets", 8, _Kind.BOOL),
_Field("enable_automatic_punctuation", 11, _Kind.BOOL),
_Field("model", 13, _Kind.STRING),
_Field("verbatim_transcripts", 14, _Kind.BOOL),
_Field("endpointing_config", 25, _Kind.MESSAGE, "EndpointingConfig", optional=True),
),
),
(
"EndpointingConfig",
(
_Field("start_history", 1, _Kind.INT32, optional=True),
_Field("start_threshold", 2, _Kind.FLOAT, optional=True),
_Field("stop_history", 3, _Kind.INT32, optional=True),
_Field("stop_threshold", 4, _Kind.FLOAT, optional=True),
_Field("stop_history_eou", 5, _Kind.INT32, optional=True),
_Field("stop_threshold_eou", 6, _Kind.FLOAT, optional=True),
),
),
(
"StreamingRecognitionConfig",
(
_Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"),
_Field("interim_results", 2, _Kind.BOOL),
),
),
(
"StreamingRecognizeRequest",
(
_Field("streaming_config", 1, _Kind.MESSAGE, "StreamingRecognitionConfig", oneof="streaming_request"),
_Field("audio_content", 2, _Kind.BYTES, oneof="streaming_request"),
),
),
(
"WordInfo",
(
_Field("start_time", 1, _Kind.INT32),
_Field("end_time", 2, _Kind.INT32),
_Field("word", 3, _Kind.STRING),
),
),
(
"SpeechRecognitionAlternative",
(
_Field("transcript", 1, _Kind.STRING),
_Field("words", 3, _Kind.MESSAGE, "WordInfo", repeated=True),
),
),
(
"StreamingRecognitionResult",
(
_Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True),
_Field("is_final", 2, _Kind.BOOL),
),
),
(
"StreamingRecognizeResponse",
(_Field("results", 1, _Kind.MESSAGE, "StreamingRecognitionResult", repeated=True),),
),
(
"RecognizeRequest",
(
_Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"),
_Field("audio", 2, _Kind.BYTES),
),
),
(
"SpeechRecognitionResult",
(_Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True),),
),
(
"RecognizeResponse",
(_Field("results", 1, _Kind.MESSAGE, "SpeechRecognitionResult", repeated=True),),
),
)
def _oneof_names(fields: tuple[_Field, ...]) -> tuple[str, ...]:
declared: Final = tuple(dict.fromkeys(field.oneof for field in fields if field.oneof))
synthetic: Final = tuple(f"_{field.name}" for field in fields if field.optional)
return declared + synthetic
def _field_proto(field: _Field, oneofs: tuple[str, ...]) -> descriptor_pb2.FieldDescriptorProto:
oneof: Final = f"_{field.name}" if field.optional else field.oneof
return descriptor_pb2.FieldDescriptorProto(
name=field.name,
number=field.number,
type=descriptor_pb2.FieldDescriptorProto.Type.Value(field.kind.value),
label=(
descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED
if field.repeated
else descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL
),
type_name=f".{_PACKAGE}.{field.type_name}" if field.type_name else None,
oneof_index=oneofs.index(oneof) if oneof else None,
proto3_optional=field.optional or None,
)
def _message_proto(name: str, fields: tuple[_Field, ...]) -> descriptor_pb2.DescriptorProto:
oneofs: Final = _oneof_names(fields)
return descriptor_pb2.DescriptorProto(
name=name,
field=[_field_proto(field, oneofs) for field in fields],
oneof_decl=[descriptor_pb2.OneofDescriptorProto(name=oneof) for oneof in oneofs],
)
_FILE: Final = descriptor_pb2.FileDescriptorProto(
name="litellm/nvidia_riva/riva_asr.proto",
package=_PACKAGE,
syntax="proto3",
enum_type=[
descriptor_pb2.EnumDescriptorProto(
name="AudioEncoding",
value=[
descriptor_pb2.EnumValueDescriptorProto(name=encoding.name, number=encoding.value)
for encoding in AudioEncoding
],
)
],
message_type=[_message_proto(name, fields) for name, fields in _MESSAGES],
)
_POOL: Final = descriptor_pool.DescriptorPool()
_POOL.AddSerializedFile(_FILE.SerializeToString())
def _message_class(name: str) -> type[Message]:
return message_factory.GetMessageClass(_POOL.FindMessageTypeByName(f"{_PACKAGE}.{name}"))
RecognitionConfig: Final = _message_class("RecognitionConfig")
EndpointingConfig: Final = _message_class("EndpointingConfig")
StreamingRecognitionConfig: Final = _message_class("StreamingRecognitionConfig")
StreamingRecognizeRequest: Final = _message_class("StreamingRecognizeRequest")
StreamingRecognizeResponse: Final = _message_class("StreamingRecognizeResponse")
RecognizeRequest: Final = _message_class("RecognizeRequest")
RecognizeResponse: Final = _message_class("RecognizeResponse")

113
litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py Normal file
View file

@ -0,0 +1,113 @@
"""
gRPC calls to NVIDIA Riva's speech recognition service.
The ``authorization`` and NVCF ``function-id`` metadata ride on each call, never on the channel, so every
request carries them exactly once over both plaintext and TLS.
"""
from collections.abc import Iterator, Mapping, Sequence
from typing import Any, Final, TypeAlias
import grpc
from google.protobuf.message import Message
from pydantic import TypeAdapter
from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema as schema
from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException
# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches
# NVIDIA's recommended chunk size for streaming ASR, small enough for
# responsive endpointing, large enough to keep per-RPC overhead low.
_CHUNK_SAMPLES: Final = 1600
_CHUNK_BYTES: Final = _CHUNK_SAMPLES * 2
CallMetadata: TypeAlias = tuple[tuple[str, str], ...]
_ENDPOINTING_FIELDS: Final = TypeAdapter(Mapping[str, object])
def call_metadata(api_key: str | None, nvcf_function_id: object) -> CallMetadata:
function_id: Final = (("function-id", str(nvcf_function_id)),) if nvcf_function_id else ()
authorization: Final = (("authorization", f"Bearer {api_key}"),) if api_key else ()
return function_id + authorization
def open_channel(api_base: str, use_ssl: bool) -> grpc.Channel:
if use_ssl:
return grpc.secure_channel(api_base, grpc.ssl_channel_credentials())
return grpc.insecure_channel(api_base)
def recognition_config(config: Mapping[str, Any]) -> Message:
endpointing: Final = _endpointing_config(config.get("endpointing_config"))
return schema.RecognitionConfig(
encoding=schema.LINEAR_PCM,
sample_rate_hertz=int(config["sample_rate_hertz"]),
language_code=config["language_code"],
audio_channel_count=int(config["audio_channel_count"]),
enable_automatic_punctuation=bool(config.get("enable_automatic_punctuation", True)),
enable_word_time_offsets=bool(config.get("enable_word_time_offsets", False)),
max_alternatives=int(config.get("max_alternatives", 1)),
model=config.get("model", "") or "",
verbatim_transcripts=bool(config.get("verbatim_transcripts", False)),
profanity_filter=bool(config.get("profanity_filter", False)),
**({} if endpointing is None else {"endpointing_config": endpointing}),
)
def _endpointing_config(endpointing: object) -> Message | None:
try:
fields: Final = _ENDPOINTING_FIELDS.validate_python(endpointing)
return schema.EndpointingConfig(**fields) if fields else None
except (ValueError, TypeError):
# A missing, unknown or mistyped field falls back to Riva's default
# endpointing rather than failing the whole request.
return None
def _streaming_requests(config: Message, pcm_bytes: bytes) -> Iterator[Message]:
yield schema.StreamingRecognizeRequest(
streaming_config=schema.StreamingRecognitionConfig(config=config, interim_results=False)
)
for offset in range(0, len(pcm_bytes), _CHUNK_BYTES):
yield schema.StreamingRecognizeRequest(audio_content=pcm_bytes[offset : offset + _CHUNK_BYTES])
def streaming_recognize(
channel: grpc.Channel,
config: Message,
pcm_bytes: bytes,
metadata: CallMetadata,
timeout: float | None,
) -> Iterator[Message]:
rpc: Final = channel.stream_stream(
schema.STREAMING_RECOGNIZE_METHOD,
request_serializer=schema.StreamingRecognizeRequest.SerializeToString,
response_deserializer=schema.StreamingRecognizeResponse.FromString,
)
return rpc(_streaming_requests(config, pcm_bytes), metadata=metadata, timeout=timeout)
def recognize(
channel: grpc.Channel,
config: Message,
pcm_bytes: bytes,
metadata: CallMetadata,
timeout: float | None,
) -> Sequence[Message]:
rpc: Final = channel.unary_unary(
schema.RECOGNIZE_METHOD,
request_serializer=schema.RecognizeRequest.SerializeToString,
response_deserializer=schema.RecognizeResponse.FromString,
)
pending: Final = rpc.future(schema.RecognizeRequest(config=config, audio=pcm_bytes), metadata=metadata)
try:
return (pending.result(timeout=timeout),)
except Exception as e:
if pending.done():
raise
pending.cancel()
raise NvidiaRivaException(
status_code=408,
message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds",
) from e

10
litellm/llms/openai/responses/transformation.py
View file

@ -669,7 +669,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
prompt_cache_options: object = None,
) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]:
validated_input: Final = self._validate_input_param(
self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
self._drop_bridge_minted_reasoning_items(input), drop_params=drops_params(litellm_params)
)
reads_breakpoints: Final = self._reads_prompt_cache_breakpoints(
model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
@ -951,7 +951,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized
def _validate_input_param(
self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
self, input: str | ResponseInputParam, *, drop_params: bool = False
) -> str | ResponseInputParam:
"""
Ensure all input fields if pydantic are converted to dict
@ -965,9 +965,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
# if it's pydantic, convert to dict
if isinstance(item, BaseModel):
validated_input.append(
replayable_responses_item(
item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by
)
replayable_responses_item(item.model_dump(exclude_none=True), drops_set_created_by=drop_params)
)
elif isinstance(item, dict):
validated_input.append(
@ -975,7 +973,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
self._handle_reasoning_item(cast(dict[str, object], item))
if item.get("type") == "reasoning"
else item,
drops_set_created_by=drops_set_created_by,
drops_set_created_by=drop_params,
)
)
else:

2
litellm/llms/perplexity/responses/transformation.py
View file

@ -89,7 +89,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
"""Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}."""
input = self._ensure_message_type(input)
if model.startswith("preset/"):
input = self._validate_input_param(input, drops_set_created_by=drops_params(litellm_params))
input = self._validate_input_param(input, drop_params=drops_params(litellm_params))
data: Final[dict] = {
"preset": model[len("preset/") :],
"input": input,

255
litellm/model_prices_and_context_window_backup.json
View file

@ -2125,6 +2125,7 @@
"anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -2213,6 +2214,7 @@
"global.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -2301,6 +2303,7 @@
"us.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2389,6 +2392,7 @@
"eu.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2472,6 +2476,7 @@
"au.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2560,6 +2565,7 @@
"jp.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2958,6 +2964,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3002,6 +3009,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3046,6 +3054,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3090,6 +3099,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3134,6 +3144,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3178,6 +3189,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -4063,6 +4075,7 @@
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"litellm_provider": "azure_ai",
@ -4204,6 +4217,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -16507,6 +16521,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -17042,6 +17057,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -21910,6 +21926,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
@ -22068,6 +22085,7 @@
"prompt_cache_min_tokens": 1024,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
@ -36308,6 +36326,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3.2e-05,
"output_cost_per_image_token_batches": 1.6e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
@ -36331,6 +36350,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3.2e-05,
"output_cost_per_image_token_batches": 1.6e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
@ -36350,6 +36370,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/images/generations",
@ -36385,6 +36406,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
@ -36420,6 +36442,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
@ -39774,6 +39797,7 @@
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_image_token": 8e-06,
"output_cost_per_image_token_batches": 4e-06,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/images/generations",
@ -50486,6 +50510,7 @@
"prompt_cache_min_tokens": 1024,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -50563,6 +50588,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -52968,6 +52994,7 @@
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_creation_input_token_cost_batches": 3.125e-06,
@ -53006,6 +53033,7 @@
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_creation_input_token_cost_batches": 3.125e-06,
@ -53255,6 +53283,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -59906,6 +59935,7 @@
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_image_token": 3.2e-05,
"output_cost_per_image_token_batches": 1.6e-05,
"output_cost_per_token": 1e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://developers.openai.com/api/docs/pricing",
@ -60629,6 +60659,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -64719,7 +64750,8 @@
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number. On mapped Claude entries from Anthropic, Azure Foundry, Bedrock and Vertex it only fills the keys an entry leaves out, so the entry's own values always win.",
"model_info": {
"mode": "chat",
"max_input_tokens": 200000,
@ -64762,6 +64794,14 @@
"thinking_always_on": true
}
},
{
"name": "claude-thinking-on-by-default",
"pattern": "claude-[a-z]+-[5-9](?!\\d)",
"description": "Claude at major 5 or higher, in any id shape that contains claude-<family>-<major>. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.",
"model_info": {
"thinking_on_by_default": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
@ -69285,6 +69325,7 @@
"output_cost_per_token": 1.2e-05,
"prompt_cache_min_tokens": 1024,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -69360,6 +69401,7 @@
"output_cost_per_token": 3e-05,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -69616,6 +69658,7 @@
"output_cost_per_token": 1.2e-05,
"prompt_cache_min_tokens": 1024,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -69681,6 +69724,7 @@
"output_cost_per_token": 3e-05,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -75432,6 +75476,7 @@
"apac.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
@ -75548,6 +75593,7 @@
"max_output_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77052,6 +77098,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77104,6 +77151,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77148,6 +77196,7 @@
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77195,6 +77244,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77231,6 +77281,7 @@
"max_output_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77287,6 +77338,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77333,6 +77385,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77373,6 +77426,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77414,6 +77468,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77460,6 +77515,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77511,6 +77567,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77557,6 +77614,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77617,6 +77675,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77668,6 +77727,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77718,6 +77778,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77762,6 +77823,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -79521,6 +79583,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -79735,6 +79798,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -79901,6 +79965,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -81886,6 +81951,7 @@
"prompt_cache_min_tokens": 512.0,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_anthropic_thinking_payload": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
@ -85912,5 +85978,192 @@
],
"supports_pdf_input": true,
"supports_vision": true
},
"bedrock/us-gov-east-1/moonshotai.kimi-k3": {
"cache_creation_input_token_cost": 4.95e-06,
"cache_creation_input_token_cost_flex": 2.475e-06,
"cache_creation_input_token_cost_priority": 8.6625e-06,
"cache_read_input_token_cost": 3.96e-07,
"cache_read_input_token_cost_flex": 1.98e-07,
"cache_read_input_token_cost_priority": 6.93e-07,
"input_cost_per_token": 3.96e-06,
"input_cost_per_token_batches": 1.98e-06,
"input_cost_per_token_flex": 1.98e-06,
"input_cost_per_token_priority": 6.93e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 868928,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.98e-05,
"output_cost_per_token_batches": 9.9e-06,
"output_cost_per_token_flex": 9.9e-06,
"output_cost_per_token_priority": 3.465e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_prompt_cache_breakpoint": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_regex_lookaround": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"token_count_framing": 64.0,
"token_count_tokenizer": "kimi_k3"
},
"bedrock/us-gov-west-1/moonshotai.kimi-k3": {
"cache_creation_input_token_cost": 4.95e-06,
"cache_creation_input_token_cost_flex": 2.475e-06,
"cache_creation_input_token_cost_priority": 8.6625e-06,
"cache_read_input_token_cost": 3.96e-07,
"cache_read_input_token_cost_flex": 1.98e-07,
"cache_read_input_token_cost_priority": 6.93e-07,
"input_cost_per_token": 3.96e-06,
"input_cost_per_token_batches": 1.98e-06,
"input_cost_per_token_flex": 1.98e-06,
"input_cost_per_token_priority": 6.93e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 868928,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.98e-05,
"output_cost_per_token_batches": 9.9e-06,
"output_cost_per_token_flex": 9.9e-06,
"output_cost_per_token_priority": 3.465e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_prompt_cache_breakpoint": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_regex_lookaround": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"token_count_framing": 64.0,
"token_count_tokenizer": "kimi_k3"
},
"bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": {
"deprecation_date": "2026-09-10",
"deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 195904,
"max_output_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json",
"successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/xai.grok-4.7": {
"cache_read_input_token_cost": 6.6e-07,
"cache_read_input_token_cost_flex": 3.3e-07,
"cache_read_input_token_cost_priority": 1.155e-06,
"input_cost_per_token": 2.64e-06,
"input_cost_per_token_batches": 1.32e-06,
"input_cost_per_token_flex": 1.32e-06,
"input_cost_per_token_priority": 4.62e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 7.92e-06,
"output_cost_per_token_batches": 3.96e-06,
"output_cost_per_token_flex": 3.96e-06,
"output_cost_per_token_priority": 1.386e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"use_openai_responses_path": true
},
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": {
"deprecation_date": "2026-09-10",
"deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 195904,
"max_output_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json",
"successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/xai.grok-4.7": {
"cache_read_input_token_cost": 6.6e-07,
"cache_read_input_token_cost_flex": 3.3e-07,
"cache_read_input_token_cost_priority": 1.155e-06,
"input_cost_per_token": 2.64e-06,
"input_cost_per_token_batches": 1.32e-06,
"input_cost_per_token_flex": 1.32e-06,
"input_cost_per_token_priority": 4.62e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 7.92e-06,
"output_cost_per_token_batches": 3.96e-06,
"output_cost_per_token_flex": 3.96e-06,
"output_cost_per_token_priority": 1.386e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"use_openai_responses_path": true
},
"bedrock_mantle/xai.grok-4.7": {
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_flex": 2.75e-07,
"cache_read_input_token_cost_priority": 9.625e-07,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_batches": 1.1e-06,
"input_cost_per_token_flex": 1.1e-06,
"input_cost_per_token_priority": 3.85e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"output_cost_per_token_batches": 3.3e-06,
"output_cost_per_token_flex": 3.3e-06,
"output_cost_per_token_priority": 1.155e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"use_openai_responses_path": true
}
}

8
litellm/proxy/_types.py
View file

@ -2480,6 +2480,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
Documents all the fields supported by `general_settings` in config.yaml
"""
use_spend_logs_ledger: bool = Field(
False,
description="Apply request accounting from stored SpendLogs deliveries. Enable only after all accounting writers are upgraded and ledger activation gates pass.",
)
request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
None,
description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.",
@ -3800,6 +3805,9 @@ class SpendLogsMetadata(TypedDict):
class SpendLogsPayload(TypedDict):
accounting_payload: NotRequired[Mapping[str, object]] # writable-ok: capture before buffering
accounting_log_disabled: NotRequired[bool] # writable-ok: capture logging mode before buffering
accounting_delivery_id: NotRequired[str] # writable-ok: assigned once before buffering
request_id: str
call_type: str
api_key: str

1
litellm/proxy/alerting_endpoints/__init__.py Normal file
View file

@ -0,0 +1 @@
"""HTTP endpoints for gateway alerts."""

109
litellm/proxy/alerting_endpoints/model_deprecation_alerts.py Normal file
View file

@ -0,0 +1,109 @@
"""Shared acknowledgement state for model migration alerts."""
import hashlib
import json
from datetime import date, datetime, timezone
from typing import Annotated, Final
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import TypeAdapter
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.deps.database import DatabaseDep
from litellm.repositories.config_repository import ConfigRepository
from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket
from litellm.repositories.table_repositories import DailyTagSpendRepository
from litellm.types.proxy.auth.user_roles import LitellmUserRoles
from litellm.types.proxy.model_deprecation import DeprecationAlertState, DeprecationAlertUpdate, DeprecationTagUsage
router: Final = APIRouter()
def alert_state_repository(database: DatabaseDep) -> ConfigRepository:
if database is None:
raise HTTPException(status_code=503, detail="Alert state requires a database connection")
return ConfigRepository(database.prisma, use_writer=True)
def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository:
if database is None:
raise HTTPException(status_code=503, detail="Model usage tags require a database connection")
return DailyTagSpendRepository(database.prisma)
def _state_key(model: str, deprecation_date: date) -> str:
identity: Final = json.dumps([model, deprecation_date.isoformat()])
digest: Final = hashlib.sha256(identity.encode()).hexdigest()
return f"model_deprecation_alert:{digest}"
def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None:
if user.user_role == LitellmUserRoles.PROXY_ADMIN:
return
if not write and user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY:
return
raise HTTPException(status_code=403, detail="Only gateway admins can manage model deprecation alerts")
@router.get("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState)
async def get_deprecation_alert_state(
model: Annotated[str, Query(min_length=1, max_length=1024)],
deprecation_date: date,
user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
repository: Annotated[ConfigRepository, Depends(alert_state_repository)],
) -> DeprecationAlertState:
_require_admin(user, write=False)
record: Final = await repository.get_param(_state_key(model, deprecation_date))
if record is None:
return DeprecationAlertState(model=model, deprecation_date=deprecation_date)
return DeprecationAlertState.model_validate(record.param_value)
@router.put("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState)
async def update_deprecation_alert_state(
data: DeprecationAlertUpdate,
user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
repository: Annotated[ConfigRepository, Depends(alert_state_repository)],
) -> DeprecationAlertState:
_require_admin(user, write=True)
state: Final = DeprecationAlertState(
**data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id
)
await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json"))
return state
@router.get("/alerting/model-deprecations/tags", tags=["alerting"], response_model=tuple[DeprecationTagUsage, ...])
async def get_deprecation_usage_tags(
model: Annotated[str, Query(min_length=1, max_length=1024)],
api_keys: Annotated[tuple[str, ...], Query(min_length=1, max_length=100)],
start_date: date,
end_date: date,
user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
repository: Annotated[DailyTagSpendRepository, Depends(alert_tag_repository)],
) -> tuple[DeprecationTagUsage, ...]:
"""Distinct tags observed with this model for a page of affected reporting key IDs."""
_require_admin(user, write=False)
if not 0 <= (end_date - start_date).days < 30:
raise HTTPException(status_code=422, detail="Select up to 30 consecutive calendar dates")
rows: Final = await repository.table.group_by(
by=["api_key", "tag"],
where={
"api_key": {
# bounded-ok: query validation limits api_keys to 100 IDs.
"in": [api_key_from_bucket(key) for key in api_keys]
},
"model": model,
"date": {"gte": start_date.isoformat(), "lte": end_date.isoformat()},
"api_requests": {"gt": 0},
"tag": {"not": None},
},
)
tags: Final = TypeAdapter(tuple[DeprecationTagUsage, ...]).validate_python(rows)
return tuple(
sorted(
(DeprecationTagUsage(api_key=api_key_bucket(usage.api_key), tag=usage.tag) for usage in tags),
key=lambda usage: (usage.api_key, usage.tag),
)
)

25
litellm/proxy/anthropic_endpoints/answered_status.py
View file

@ -1,10 +1,29 @@
from typing import Final, TypeVar
from litellm.exceptions import InternalServerError, MidStreamFallbackError
from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError
_PROVIDER_OVERLOAD_STATUS_CODE: Final = 529
_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE: Final = 503
_Raised = TypeVar("_Raised", bound=Exception)
def is_provider_overload(exc: Exception) -> bool:
return _provider_status_code(exc) == _PROVIDER_OVERLOAD_STATUS_CODE
def with_openai_overload_status(exc: _Raised) -> _Raised:
return (
_copy_carrying(
_copy_carrying(exc, "provider_status_code", _PROVIDER_OVERLOAD_STATUS_CODE),
"status_code",
_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE,
)
if is_provider_overload(exc)
else exc
)
def _provider_status_code(exc: Exception) -> int | None:
unwrapped = exc # rebind-ok: the recursion gate requires iterative wrapper traversal
while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
@ -12,6 +31,10 @@ def _provider_status_code(exc: Exception) -> int | None:
match unwrapped:
case InternalServerError(provider_status_code=int() as provider_status):
return provider_status
case APIError(provider_status_code=int() as provider_status):
return provider_status
case APIError(status_code=int() as provider_status):
return provider_status
case _:
return None

78
litellm/proxy/common_request_processing.py
View file

@ -98,6 +98,7 @@ from litellm.litellm_core_utils.streaming_handler import (
)
from litellm.litellm_core_utils.tool_fields import DropPolicy, MalformedField, drop_policy, resolve_tool_fields
from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload, with_openai_overload_status
from litellm.proxy.auth.auth_checks import (
can_key_call_resolved_model,
is_model_cost_zero,
@ -118,6 +119,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
)
from litellm.proxy.common_utils.openai_error_payload import (
LITELLM_CALL_ID_HEADER,
OverloadedProxyException,
attribute_of,
error_status_code,
openai_error_param,
@ -167,6 +169,7 @@ _ChunkT: Final = TypeVar("_ChunkT")
KNOWN_PROXY_ROUTES: Final = frozenset(
route for member in LiteLLMRoutes for route in member.value if route.startswith("/")
)
_OPENAI_OVERLOAD_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion", "aresponses"})
ProxyRouteType: TypeAlias = Literal[
"acompletion",
@ -4409,28 +4412,28 @@ class ProxyBaseLLMRequestProcessing:
version: str | None = None,
) -> Never:
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
log_llm_api_exception(e, self.litellm_call_id)
answered: Final = with_openai_overload_status(e) if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES else e
log_llm_api_exception(answered, self.litellm_call_id)
billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj")
failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e)
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
original_exception=answered,
request_data=self.data,
dispatched_call_type=self.route_type,
)
# Use transformed exception if callback returned one, otherwise use original
if transformed_exception is not None:
e = transformed_exception
litellm_debug_info: Final = getattr(e, "litellm_debug_info", "")
failure: Final = transformed_exception if transformed_exception is not None else answered
litellm_debug_info: Final = getattr(failure, "litellm_debug_info", "")
verbose_proxy_logger.debug(
"An error occurred: %s %s",
e,
failure,
litellm_debug_info,
)
timeout: Final = getattr(
e, "timeout", None
failure, "timeout", None
) # returns the timeout set by the wrapper. Used for testing if model-specific timeout are set correctly
_litellm_logging_obj: Final = (
billing_logging_obj if isinstance(billing_logging_obj, LiteLLMLoggingObj) else None
@ -4462,11 +4465,11 @@ class ProxyBaseLLMRequestProcessing:
# a failed request reports no timing, matching /v1/chat/completions
read_timing_from_logging_obj=False,
)
headers = getattr(e, "headers", None) or {}
headers = getattr(failure, "headers", None) or {}
if not headers:
_response: Final = attribute_of(e, "response")
_response: Final = attribute_of(failure, "response")
_response_headers: Final = getattr(_response, "headers", None) if _response is not None else None
_provider_headers: Final = _response_headers or getattr(e, "litellm_response_headers", None)
_provider_headers: Final = _response_headers or getattr(failure, "litellm_response_headers", None)
if _provider_headers:
headers = get_response_headers(dict(_provider_headers))
headers.update(custom_headers)
@ -4486,21 +4489,21 @@ class ProxyBaseLLMRequestProcessing:
safe_headers: Final = {k: v for k, v in headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS}
self._apply_router_cooldown_retry_after(safe_headers, e)
self._apply_router_cooldown_retry_after(safe_headers, failure)
if isinstance(e, ProxyException):
e.headers = {
**{k: v for k, v in e.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS},
if isinstance(failure, ProxyException):
failure.headers = {
**{k: v for k, v in failure.headers.items() if k.lower() not in UNSAFE_PROXY_RESPONSE_HEADERS},
**{k: v if isinstance(v, str) else str(v) for k, v in safe_headers.items()},
}
raise e
raise failure
if isinstance(e, HTTPException):
raise proxy_exception_from_http_exception(e, safe_headers)
elif isinstance(e, httpx.HTTPStatusError):
if isinstance(failure, HTTPException):
raise proxy_exception_from_http_exception(failure, safe_headers)
elif isinstance(failure, httpx.HTTPStatusError):
# Handle httpx.HTTPStatusError - extract actual error from response
# This matches the original behavior before the refactor in commit 511d435f6f
http_status_error: Final[httpx.HTTPStatusError] = e
http_status_error: Final[httpx.HTTPStatusError] = failure
error_body: Final = await http_status_error.response.aread()
error_text: Final = error_body.decode("utf-8")
@ -4510,12 +4513,12 @@ class ProxyBaseLLMRequestProcessing:
detail={"error": error_text},
headers=error_headers,
)
error_msg: Final = f"{e}"
error_msg: Final = f"{failure}"
# Check for AttributeError in the exception chain.
# The AttributeError may be wrapped in multiple layers
# (e.g. AttributeError -> OpenAIException -> APIConnectionError),
# so walk __cause__, __context__, and original_exception recursively.
has_attribute_error: Final = _has_attribute_error_in_chain(e)
has_attribute_error: Final = _has_attribute_error_in_chain(failure)
if has_attribute_error:
raise ProxyException(
@ -4530,12 +4533,12 @@ class ProxyBaseLLMRequestProcessing:
# VertexAIError, etc.) all have a status_code attribute reflecting
# the upstream API response. Use it to return the correct HTTP code
# instead of defaulting to 500.
_exc_status_code: Final = getattr(e, "status_code", None)
_exc_status_code: Final = getattr(failure, "status_code", None)
if _exc_status_code is not None and isinstance(_exc_status_code, int) and 400 <= _exc_status_code <= 599:
_code = _exc_status_code
else:
_code = status.HTTP_500_INTERNAL_SERVER_ERROR
if should_report_bug(e):
if should_report_bug(failure):
proxy_server_request: Final = self.data.get("proxy_server_request")
request_url: Final = (
proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None
@ -4544,24 +4547,33 @@ class ProxyBaseLLMRequestProcessing:
verbose_proxy_logger.error(
bug_report_notice(
build_proxy_bug_report(
e,
failure,
call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES),
custom_llm_provider=self.data.get("custom_llm_provider"),
stream=self.data.get("stream"),
)
)
)
client_message: Final = getattr(e, "message", error_msg)
client_message: Final = getattr(failure, "message", error_msg)
redacted_client_message: Final = redact_internal_details_from_client_message(
strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg
)
provider_specific_fields: Final = getattr(
failure, "provider_specific_fields", None
) or relayed_provider_specific_fields(failure)
if self.route_type in _OPENAI_OVERLOAD_ROUTE_TYPES and is_provider_overload(failure):
raise OverloadedProxyException(
message=redacted_client_message,
headers=cast(dict[str, str], safe_headers), # cast-ok: ProxyException coerces header values to strings
provider_specific_fields=provider_specific_fields,
)
raise ProxyException(
message=redact_internal_details_from_client_message(
strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg
),
type=openai_error_type(e, _code),
param=openai_error_param(e),
openai_code=getattr(e, "code", None),
message=redacted_client_message,
type=openai_error_type(failure, _code),
param=openai_error_param(failure),
openai_code=getattr(failure, "code", None),
code=_code,
provider_specific_fields=getattr(e, "provider_specific_fields", None)
or relayed_provider_specific_fields(e),
provider_specific_fields=provider_specific_fields,
headers=safe_headers,
)

46
litellm/proxy/common_utils/model_deprecation.py
View file

@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import date, datetime, timezone
@ -166,14 +167,20 @@ def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus:
return "upcoming"
def _checked_successor(model_name: str, model_info: Mapping[str, object], llm_router: Router) -> str | None:
def _checked_successor(
model_name: str,
model_info: Mapping[str, object],
llm_router: Router,
successor_log_level: int,
) -> str | None:
"""The deployment's successor_model as set, with a warning when no model in its team scope matches it"""
successor: Final = _successor_model(model_info.get("successor_model"))
if successor is None:
return None
team_id: Final = model_info.get("team_id")
if not llm_router.registry().deployments(successor, team_id=team_id if isinstance(team_id, str) else None):
verbose_logger.warning(
verbose_logger.log(
successor_log_level,
"model_deprecation: %s names successor %s, which matches no model group, alias, or wildcard route here",
model_name,
successor,
@ -187,6 +194,7 @@ def _build_info(
today: date,
warn_within_days: int,
catalog: Mapping[str, object],
successor_log_level: int,
) -> _DeploymentInfo | None:
model_name: Final = deployment.get("model_name")
if not isinstance(model_name, str) or not model_name:
@ -197,10 +205,18 @@ def _build_info(
return None
days_until: Final = (resolved.deprecation_date - today).days
configured: Final = _checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router)
model_info: Final = _mapping_field(deployment, "model_info")
configured: Final = _checked_successor(model_name, model_info, llm_router, successor_log_level)
team_public_model_name_value: Final = model_info.get("team_public_model_name")
team_public_model_name: Final = (
team_public_model_name_value
if isinstance(team_public_model_name_value, str) and team_public_model_name_value.strip()
else None
)
return _DeploymentInfo(
info=ModelDeprecationInfo(
model_name=model_name,
team_public_model_name=team_public_model_name,
litellm_model=resolved.litellm_model,
deprecation_date=resolved.deprecation_date,
days_until_deprecation=days_until,
@ -218,7 +234,7 @@ class _DeploymentInfo:
configured_successor: bool
def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) -> ModelDeprecationInfo:
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names,
a successor configured in model_info ahead of one the cost map supplies"""
first: Final = group[0].info
@ -234,7 +250,8 @@ def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
if not successors:
return first
if len(frozenset(successors)) > 1:
verbose_logger.warning(
verbose_logger.log(
successor_log_level,
"model_deprecation: %s names several successors %s, reporting %s",
first.model_name,
sorted(frozenset(successors)),
@ -245,11 +262,12 @@ def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
def _dedupe(
models: Sequence[_DeploymentInfo],
successor_log_level: int,
) -> tuple[ModelDeprecationInfo, ...]:
"""Report a model group carrying the same date on several deployments once"""
ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date))
return tuple(
_merge_group(tuple(group))
_merge_group(tuple(group), successor_log_level)
for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date))
)
@ -264,11 +282,18 @@ def _bucket(models: Sequence[ModelDeprecationInfo], status: DeprecationStatus) -
def _collect_from_router(
llm_router: Router, today: date, warn_within_days: int, catalog: Mapping[str, object]
llm_router: Router,
today: date,
warn_within_days: int,
catalog: Mapping[str, object],
successor_log_level: int,
) -> tuple[ModelDeprecationInfo, ...]:
deployments: Final = llm_router.registry().deployments() or ()
infos: Final = (_build_info(deployment, llm_router, today, warn_within_days, catalog) for deployment in deployments)
deduped: Final = _dedupe(tuple(info for info in infos if info is not None))
infos: Final = (
_build_info(deployment, llm_router, today, warn_within_days, catalog, successor_log_level)
for deployment in deployments
)
deduped: Final = _dedupe(tuple(info for info in infos if info is not None), successor_log_level)
verbose_logger.debug(
"model_deprecation: %d/%d deployments carry a deprecation date",
@ -283,12 +308,13 @@ def collect_model_deprecations(
warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS,
today: date | None = None,
loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map,
successor_log_level: int = logging.WARNING,
) -> ModelDeprecationResponse:
"""Bucket every deployment carrying a deprecation date by how urgent it is"""
snapshot_time: Final = datetime.now(timezone.utc)
effective_today: Final = today or snapshot_time.date()
deduped: Final = (
_collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog())
_collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog(), successor_log_level)
if llm_router is not None
else ()
)

26
litellm/proxy/common_utils/openai_error_payload.py
View file

@ -17,6 +17,7 @@ from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
_SERVER_IS_OVERLOADED_CODE: Final = "server_is_overloaded"
_OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
{
@ -44,6 +45,31 @@ _FAILOVER_REFUSALS: Final = (
)
class OverloadedProxyException(ProxyException):
def __init__(
self,
message: str,
headers: dict[str, str] | None, # mutable-ok: matches ProxyException's constructor
provider_specific_fields: dict[str, object] | None, # mutable-ok: matches ProxyException's constructor
) -> None:
super().__init__( # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments
message=message,
type="service_unavailable_error",
param=None,
code=status.HTTP_503_SERVICE_UNAVAILABLE,
headers=None if headers is None else dict(headers),
provider_specific_fields=provider_specific_fields,
)
def to_dict(self) -> dict[str, object]: # mutable-ok: matches ProxyException.to_dict
return {
**_ERROR_BODY.validate_python(
super().to_dict() # pyright: ignore[reportUnknownMemberType] # ProxyException omits generic arguments
),
"code": _SERVER_IS_OVERLOADED_CODE,
}
def attribute_of(value: object, name: str, default: object = None) -> object:
return getattr(value, name, default)

98
litellm/proxy/db/budget_window_spend_writer.py
View file

@ -14,11 +14,13 @@ interval of increments queued elsewhere: the same lag the SpendLogs aggregate
it replaces (and every other spend column) already has.
"""
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from contextlib import nullcontext
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Final, Protocol
from pydantic import TypeAdapter
from typing_extensions import LiteralString, ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -74,30 +76,44 @@ _OPEN_WINDOW_SPEND_SQL: Final = (
"WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
)
_HAS_DELIVERY: Final = 'EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id)'
_SETTLED_SPEND_LOG_FILTER: Final = (
' AND NOT EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery '
'WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id '
"AND delivery.state IN ('pending', 'parked'))"
)
_SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, "
"COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
"COALESCE(SUM(spend) FILTER (WHERE "
+ _HAS_DELIVERY
+ " OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) "
"AS before_batch "
'FROM "LiteLLM_SpendLogs" '
"WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
"WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER
)
_SEED_FROM_SPEND_LOGS_TEAM_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, "
"COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
"COALESCE(SUM(spend) FILTER (WHERE "
+ _HAS_DELIVERY
+ " OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) "
"AS before_batch "
'FROM "LiteLLM_SpendLogs" '
"WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
"WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER
)
_SEED_FROM_SPEND_LOGS_KEY_UNBOUNDED_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch "
'FROM "LiteLLM_SpendLogs" '
"WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
"WHERE api_key = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER
)
_SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL: Final = (
"SELECT COALESCE(SUM(spend), 0.0) AS total, COALESCE(SUM(spend), 0.0) AS before_batch "
'FROM "LiteLLM_SpendLogs" '
"WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')"
"WHERE team_id = $1 AND \"startTime\" >= ($2::timestamptz AT TIME ZONE 'UTC')" + _SETTLED_SPEND_LOG_FILTER
)
_UPSERT_TRANSACTION_TIMEOUT: Final = timedelta(seconds=60)
@ -114,6 +130,10 @@ class _ExistingKeyRow(TypedDict):
window_duration: ReadOnly[str]
_SEED_TOTALS_ROWS: Final = TypeAdapter(tuple[_SeedTotalsRow, ...])
_EXISTING_KEY_ROWS: Final = TypeAdapter(tuple[_ExistingKeyRow, ...])
class _SeedTotalsQuery(Protocol):
async def __call__(self, query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]: ...
@ -135,6 +155,8 @@ class _UpsertBatchManager(Protocol):
class _UpsertTransaction(Protocol):
def batch_(self) -> _UpsertBatchManager: ...
async def query_raw(self, query: LiteralString, *args: object) -> Sequence[Mapping[str, object]]: ...
class _UpsertTransactionManager(Protocol):
async def __aenter__(self) -> _UpsertTransaction: ...
@ -178,6 +200,7 @@ class WindowSpendLogsAggregate(Protocol):
entity_id: str,
window_start: datetime,
batch_started_at: datetime | None,
query: _SeedTotalsQuery | None = None,
) -> WindowSeedTotals | None: ...
@ -187,6 +210,7 @@ async def spend_logs_seed_totals(
entity_id: str,
window_start: datetime,
batch_started_at: datetime | None,
query: _SeedTotalsQuery | None = None,
) -> WindowSeedTotals | None:
"""LiteLLM_SpendLogs spend for one entity since window_start, both in full
and up to the start of the batch being flushed, in one scan.
@ -210,11 +234,12 @@ async def spend_logs_seed_totals(
bounded_sql, unbounded_sql = _SEED_FROM_SPEND_LOGS_TEAM_SQL, _SEED_FROM_SPEND_LOGS_TEAM_UNBOUNDED_SQL
else:
return None
read: Final = query or _seed_totals_query(prisma_client)
async with db_span("seed_window_spend", "LiteLLM_SpendLogs"):
rows: Final = (
await _seed_totals_query(prisma_client)(unbounded_sql, entity_id, window_start)
await read(unbounded_sql, entity_id, window_start)
if batch_started_at is None
else await _seed_totals_query(prisma_client)(
else await read(
bounded_sql,
entity_id,
window_start,
@ -246,9 +271,10 @@ def _primary_key(transaction: WindowSpendTransaction) -> tuple[str, str, str]:
async def _existing_primary_keys(
prisma_client: "PrismaClient",
transactions: tuple[WindowSpendTransaction, ...],
query: _ExistingKeysQuery | None = None,
) -> frozenset[tuple[str, str, str]]:
async with db_span("select_window_spend_rows", "LiteLLM_BudgetWindowSpend"):
rows: Final = await _existing_keys_query(prisma_client)(
rows: Final = await (query or _existing_keys_query(prisma_client))(
_SELECT_EXISTING_ROWS_SQL,
tuple(transaction["entity_type"] for transaction in transactions),
tuple(transaction["entity_id"] for transaction in transactions),
@ -262,6 +288,7 @@ async def _seed_base_for_missing_row(
transaction: WindowSpendTransaction,
existing_primary_keys: frozenset[tuple[str, str, str]],
spend_logs_aggregate: WindowSpendLogsAggregate,
query: _SeedTotalsQuery | None = None,
) -> float:
"""Spend already recorded for a window that has no row yet.
@ -272,12 +299,24 @@ async def _seed_base_for_missing_row(
"""
if _primary_key(transaction) in existing_primary_keys:
return 0.0
totals: Final = await spend_logs_aggregate(
prisma_client=prisma_client,
entity_type=transaction["entity_type"],
entity_id=transaction["entity_id"],
window_start=datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc),
batch_started_at=_transaction_started_at(transaction),
window_start: Final = datetime.fromisoformat(transaction["window_start"]).replace(tzinfo=timezone.utc)
totals: Final = (
await spend_logs_aggregate(
prisma_client=prisma_client,
entity_type=transaction["entity_type"],
entity_id=transaction["entity_id"],
window_start=window_start,
batch_started_at=_transaction_started_at(transaction),
)
if query is None
else await spend_logs_aggregate(
prisma_client=prisma_client,
entity_type=transaction["entity_type"],
entity_id=transaction["entity_id"],
window_start=window_start,
batch_started_at=_transaction_started_at(transaction),
query=query,
)
)
if totals is None:
return 0.0
@ -324,10 +363,30 @@ def _upsert_params(
)
def _existing_keys_in(transaction: _UpsertTransaction) -> _ExistingKeysQuery:
async def query_raw(query: LiteralString, *args: object) -> Sequence[_ExistingKeyRow]:
return _EXISTING_KEY_ROWS.validate_python(await transaction.query_raw(query, *args))
return query_raw
def _seed_totals_in(transaction: _UpsertTransaction) -> _SeedTotalsQuery:
async def query_raw(query: LiteralString, *args: object) -> Sequence[_SeedTotalsRow]:
return _SEED_TOTALS_ROWS.validate_python(await transaction.query_raw(query, *args))
return query_raw
def _transaction_reads(transaction: _UpsertTransaction) -> tuple[_ExistingKeysQuery, _SeedTotalsQuery]:
"""Reads made inside an enclosing transaction share its connection, so they cannot wait on it."""
return _existing_keys_in(transaction), _seed_totals_in(transaction)
async def commit_window_spend_updates(
prisma_client: "PrismaClient",
transactions: Sequence[WindowSpendTransaction],
spend_logs_aggregate: WindowSpendLogsAggregate = spend_logs_seed_totals,
outer_transaction: _UpsertTransaction | None = None,
) -> None:
"""Apply aggregated window increments to LiteLLM_BudgetWindowSpend.
@ -343,9 +402,11 @@ async def commit_window_spend_updates(
return
ordered: Final = tuple(sorted(transactions, key=window_spend_group_key))
reads: Final = _transaction_reads(outer_transaction) if outer_transaction is not None else (None, None)
existing_primary_keys: Final = await _existing_primary_keys(
prisma_client=prisma_client,
transactions=ordered,
query=reads[0],
)
seed_bases: Final = tuple(
[
@ -354,6 +415,7 @@ async def commit_window_spend_updates(
transaction=transaction,
existing_primary_keys=existing_primary_keys,
spend_logs_aggregate=spend_logs_aggregate,
query=reads[1],
)
for transaction in ordered
]
@ -367,7 +429,9 @@ async def commit_window_spend_updates(
)
async with (
db_span("commit_window_spend_updates", "LiteLLM_BudgetWindowSpend"),
_upsert_transaction(prisma_client) as db_transaction,
(
nullcontext(outer_transaction) if outer_transaction is not None else _upsert_transaction(prisma_client)
) as db_transaction,
db_transaction.batch_() as batcher,
):
for transaction, seed_base in zip(ordered, seed_bases):

4
litellm/proxy/db/daily_spend_bulk_upsert.py
View file

@ -55,7 +55,7 @@ _ENTITY_INPUT_KEYS: Final[Mapping[DailySpendEntity, str]] = MappingProxyType(
"tag": "request_tags",
}
)
_TAGS: Final = TypeAdapter(tuple[str, ...])
_TAGS: Final = TypeAdapter(tuple[object, ...])
def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEntity) -> tuple[str | None, ...]:
@ -67,7 +67,7 @@ def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEnti
if value is None:
return ()
tags: Final = _TAGS.validate_json(value) if isinstance(value, str) else _TAGS.validate_python(value)
return tuple(dict.fromkeys(tags))
return tuple(dict.fromkeys(tag for tag in tags if isinstance(tag, str)))
if value is None:
return (None,) if entity == "user" else ()
if not isinstance(value, str) or (entity == "end_user" and not value):

306
litellm/proxy/db/db_spend_update_writer.py
View file

@ -12,6 +12,7 @@ import json
import os
import random
import time
import uuid
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
from contextlib import asynccontextmanager
from contextvars import ContextVar
@ -96,6 +97,7 @@ from litellm.types.utils import CallTypes
from litellm.types.videos.utils import VIDEO_COST_REQUEST_ID_SUFFIX, VIDEO_STATUS_CALL_TYPES
if TYPE_CHECKING:
from litellm.accounting.projection import Contribution
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -160,6 +162,18 @@ def _org_member_transaction_key(org_id: str, user_id: str) -> str:
return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
ACCOUNTING_LEDGER_KWARG: Final = "accounting_ledger"
def _persistable_api_key(token: str | None, logged_api_key: str) -> str | None:
from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash
from litellm.proxy.spend_tracking.spend_tracking_utils import redact_logged_api_key
if token is None or token == logged_api_key or is_valid_sha256_hash(token):
return token
return redact_logged_api_key(token)
async def _track_guardrail_usage_of_cost_row(
payload: SpendLogsPayload, prisma_client: "PrismaClient", disable_spend_logs: bool
) -> None:
@ -200,12 +214,20 @@ _COST_CLAIM_FIELDS: Final = frozenset({"request_id", "call_type", "spend", "star
def _cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool) -> Mapping[str, object]:
"""Reduce a job's cost row to what tells its polls apart when logging is off.
A proxy run with spend logs disabled still needs one row per job to charge it once,
A legacy proxy run with spend logs disabled still needs one row per job to charge it once,
so the row is written either way, but it carries no request of its own: no metadata,
no requester IP, no key, model, or token counts (LIT-7048).
no requester IP, no key, model, or token counts (LIT-7048). A ledger batch is deduplicated
by its delivery row instead, so it writes no local row when logging is off.
"""
if disable_spend_logs is False:
return payload
if "accounting_payload" in payload:
return {
"request_id": payload["request_id"],
"call_type": payload["call_type"],
"accounting_payload": payload["accounting_payload"],
"accounting_log_disabled": True,
}
return MappingProxyType({field: value for field, value in payload.items() if field in _COST_CLAIM_FIELDS})
@ -219,8 +241,17 @@ class _MemberSpendRow(TypedDict):
cost: ReadOnly[float]
class SpendUpsertTable(Protocol):
def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
class _SpendBatch(Protocol):
litellm_usertable: BatchTable
litellm_endusertable: SpendUpsertTable
litellm_dailymodelusage: SpendUpsertTable
def execute_raw(self, query: LiteralString, *args: object) -> None: ...
litellm_verificationtoken: BatchTable
litellm_teamtable: BatchTable
litellm_teammembership: BatchTable
@ -251,14 +282,14 @@ def _entity_spend_table(batcher: _SpendBatch, table_accessor: _EntitySpendTable)
return _ENTITY_SPEND_TABLES[table_accessor](batcher)
class _SpendBatchManager(Protocol):
class SpendBatchManager(Protocol):
async def __aenter__(self) -> _SpendBatch: ...
async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
class _SpendTransaction(Protocol):
def batch_(self) -> _SpendBatchManager: ...
def batch_(self) -> SpendBatchManager: ...
async def execute_raw(self, query: LiteralString, *args: object) -> int: ...
@ -413,8 +444,14 @@ _ENTITY_SPEND_MODELS: Final[Mapping[_EntitySpendTable, str]] = MappingProxyType(
@asynccontextmanager
async def _spend_update_tx(
prisma_client: PrismaClient, table: str, call_type: str = "commit_spend_updates"
prisma_client: PrismaClient,
table: str,
call_type: str = "commit_spend_updates",
outer_transaction: _SpendTransaction | None = None,
) -> AsyncGenerator[_SpendTransaction]:
if outer_transaction is not None:
yield outer_transaction
return
tx: Final[_SpendTransactionManager] = prisma_client.db.tx(timeout=timedelta(seconds=60))
async with db_span(call_type, table), tx as transaction:
await apply_rollup_lock_timeout(transaction)
@ -601,6 +638,21 @@ def debitable_model_access_groups(
return tuple(group for group in ordered if group in served)
def _billable_cost(response_cost: float | None, payload: SpendLogsPayload) -> float | None:
billable_cost: Final = finite_cost(response_cost)
if response_cost is not None and billable_cost is None:
verbose_proxy_logger.warning(
"Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
response_cost,
single_log_line(payload["request_id"]),
)
return billable_cost
def _iso_timestamp(value: datetime | str) -> str:
return value.isoformat() if isinstance(value, datetime) else value
class DBSpendUpdateWriter:
"""
Module responsible for
@ -649,6 +701,7 @@ class DBSpendUpdateWriter:
so the caller leaves the key, team, and user counters alone (LIT-7048).
"""
from litellm.proxy.proxy_server import (
accounting_window_keys,
disable_spend_logs,
general_settings, # pyright: ignore[reportUnknownVariableType] # legacy proxy configuration is narrowed below
litellm_proxy_budget_name,
@ -656,6 +709,13 @@ class DBSpendUpdateWriter:
)
from litellm.proxy.utils import ProxyUpdateSpend, hash_token
proxy_settings: Final = cast( # cast-ok: proxy settings use string keys
Mapping[str, object], general_settings
)
use_ledger: Final = proxy_settings.get("use_spend_logs_ledger") is True
request_fields: Final = _SPEND_METADATA_ADAPTER.validate_python(kwargs or {})
if kwargs is not None:
kwargs[ACCOUNTING_LEDGER_KWARG] = use_ledger # rebind-ok: spend counters leave ledger windows to the worker
try:
verbose_proxy_logger.debug(
"Enters prisma db call, response_cost: %s, token: %s; user_id: %s; team_id: %s",
@ -666,10 +726,9 @@ class DBSpendUpdateWriter:
)
if ProxyUpdateSpend.disable_spend_updates() is True:
return True
if token is not None and isinstance(token, str) and token.startswith("sk-"):
hashed_token = hash_token(token=token)
else:
hashed_token = token
hashed_token: Final = (
hash_token(token=token) if isinstance(token, str) and token.startswith("sk-") else token
)
## CREATE SPEND LOG PAYLOAD ##
from litellm.proxy.spend_tracking.spend_tracking_utils import (
@ -685,32 +744,42 @@ class DBSpendUpdateWriter:
end_time=end_time,
llm_router=llm_router,
)
billable_cost: Final = finite_cost(response_cost)
if response_cost is not None and billable_cost is None:
verbose_proxy_logger.warning(
"Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s",
response_cost,
single_log_line(payload["request_id"]),
)
billable_cost: Final = _billable_cost(response_cost, payload)
payload["spend"] = billable_cost or 0.0
if isinstance(payload["startTime"], datetime):
payload["startTime"] = payload["startTime"].isoformat()
if isinstance(payload["endTime"], datetime):
payload["endTime"] = payload["endTime"].isoformat()
if org_id is not None and org_id != "":
payload["startTime"] = _iso_timestamp(payload["startTime"])
payload["endTime"] = _iso_timestamp(payload["endTime"])
if org_id:
payload["organization_id"] = org_id
if team_id is not None and team_id != "":
if team_id:
payload["team_id"] = team_id
error_log_settings: Final = cast( # cast-ok: proxy settings use string keys
Mapping[str, object], general_settings
)
if use_ledger:
from litellm.accounting.snapshot import build_accounting_payload
if not _is_settle_once_cost_row(payload):
payload["accounting_delivery_id"] = uuid.uuid4().hex
payload["accounting_payload"] = build_accounting_payload(
payload,
entities={
"api_key": _persistable_api_key(hashed_token, payload.get("api_key") or ""),
"user": user_id,
"end_user": end_user_id,
"team_id": team_id,
"organization_id": org_id,
"project_id": project_id,
"agent_id": payload["agent_id"],
"proxy_budget_name": litellm_proxy_budget_name if litellm.max_budget > 0 else None,
},
model_access_groups=debitable_model_access_groups(
get_request_model_access_groups(request_fields), payload["model_id"], llm_router
),
windows=await accounting_window_keys(hashed_token, team_id) if billable_cost else (),
router=get_llm_router,
)
keep_spend_logs: Final = disable_spend_logs is False and not (
"status" in payload
and payload["status"] == "failure"
and error_log_settings.get("disable_error_logs") is True
and proxy_settings.get("disable_error_logs") is True
)
if not await self._record_spend_log(
payload=payload,
@ -739,24 +808,26 @@ class DBSpendUpdateWriter:
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
)
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=billable_cost,
user_id=user_id,
hashed_token=hashed_token,
team_id=team_id,
org_id=org_id,
project_id=project_id,
end_user_id=end_user_id,
prisma_client=prisma_client,
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
request_model_access_groups=get_request_model_access_groups(kwargs),
),
task_name="spend rollups",
)
if not use_ledger:
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=billable_cost,
user_id=user_id,
hashed_token=hashed_token,
team_id=team_id,
org_id=org_id,
project_id=project_id,
end_user_id=end_user_id,
prisma_client=prisma_client,
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
request_model_access_groups=get_request_model_access_groups(request_fields),
),
task_name="spend rollups",
)
else:
self._record_request_error_safely(payload=payload, prisma_client=prisma_client)
self._enqueue_tool_registry_upsert(
kwargs=kwargs,
completion_response=completion_response,
@ -791,7 +862,13 @@ class DBSpendUpdateWriter:
return await self._claim_settle_once_cost_row(
payload=payload, prisma_client=prisma_client, disable_spend_logs=disable_spend_logs
)
if disable_spend_logs is False:
if "accounting_payload" in payload:
await self._insert_spend_log_to_db(
payload={**payload, "accounting_log_disabled": disable_spend_logs},
prisma_client=prisma_client,
llm_router=llm_router,
)
elif disable_spend_logs is False:
await self._insert_spend_log_to_db(payload=payload, prisma_client=prisma_client, llm_router=llm_router)
return True
@ -804,7 +881,9 @@ class DBSpendUpdateWriter:
the charge and every later one finds the row and charges nothing (LIT-7048). Only
a row that recorded a charge counts: a failed poll, a request whose client
picked the job id as its call id, and the $0 row a poll left behind
while the job was still running all leave the charge to be made.
while the job was still running all leave the charge to be made. A ledger job
is owned by its delivery row, keyed by its cost row id, in every logging mode, and a
$0 poll records no delivery so it cannot block the poll that carries the cost.
"""
from litellm.repositories.table_repositories import SpendLogsRepository
@ -817,13 +896,24 @@ class DBSpendUpdateWriter:
row: Final = spend_log_row_with_retained_metadata(
_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields()
)
spend_logs: Final = SpendLogsRepository(prisma_client).table
repository: Final = SpendLogsRepository(prisma_client)
try:
claimed: Final = await spend_logs.create_many(
if "accounting_payload" in payload:
if payload["spend"] <= 0:
return True
accepted: Final = await repository.insert_with_deliveries([prisma_client.jsonify_object(row)]) == 1
if accepted:
await _track_guardrail_usage_of_cost_row(payload, prisma_client, disable_spend_logs)
return accepted
if await repository.has_delivery(request_id):
return False
claimed: Final = await repository.table.create_many(
data=[prisma_client.jsonify_object(row)],
skip_duplicates=True,
)
existing: Final = None if claimed == 1 else await spend_logs.find_unique(where={"request_id": request_id})
existing: Final = (
None if claimed == 1 else await repository.table.find_unique(where={"request_id": request_id})
)
except Exception as e: # noqa: BLE001 # prisma raises its own hierarchy; an unreachable DB queues the row like any other spend log
verbose_proxy_logger.warning(
"Could not claim spend row %s for a job's cost, queueing it: %s", request_id, e
@ -1268,10 +1358,7 @@ class DBSpendUpdateWriter:
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
try:
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
self._record_request_error_safely(payload=payload_copy, prisma_client=prisma_client)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_org_transaction",
self.add_spend_log_transaction_to_daily_org_transaction(
@ -1601,7 +1688,7 @@ class DBSpendUpdateWriter:
async def _insert_spend_log_to_db(
self,
payload: dict | SpendLogsPayload,
payload: Mapping[str, object],
prisma_client: PrismaClient | None = None,
spend_logs_url: str | None = os.getenv("SPEND_LOGS_URL"),
llm_router: "Router | None" = None,
@ -1645,6 +1732,13 @@ class DBSpendUpdateWriter:
else:
- Regular flow of this method
"""
from litellm.accounting.worker import flush_accounting_deliveries
try:
await flush_accounting_deliveries(self, prisma_client, proxy_logging_obj)
except Exception: # noqa: BLE001 # pending deliveries survive outages while legacy queues must still drain
verbose_proxy_logger.exception("Accounting delivery flush failed; pending work will retry")
if RedisUpdateBuffer.should_commit_spend_updates_to_redis():
await self._commit_spend_updates_to_db_with_redis(
prisma_client=prisma_client,
@ -2171,6 +2265,39 @@ class DBSpendUpdateWriter:
_handle_spend_update_failure = handle_spend_update_failure
@staticmethod
async def invalidate_accounting_entities(
contributions: tuple["Contribution", ...], proxy_logging_obj: ProxyLogging
) -> None:
await DBSpendUpdateWriter._invalidate_project_caches(
tuple(dict(item.key)["project_id"] for item in contributions if item.table == "LiteLLM_ProjectTable"),
proxy_logging_obj,
)
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
details: Final = _SPEND_METADATA_ADAPTER.validate_python(
proxy_logging_obj.call_details # pyright: ignore[reportUnknownMemberType] # legacy logging metadata is validated at this boundary
)
user_api_key_cache: Final = details.get("user_api_key_cache")
if isinstance(user_api_key_cache, UserApiKeyCache):
with service_target(AUTH_OBJECTS_TARGET):
for item in contributions:
if item.table == "LiteLLM_TeamMembership":
await user_api_key_cache.async_delete_cache(
key=f"team_membership:{dict(item.key)['user_id']}:{dict(item.key)['team_id']}"
)
async def apply_accounting_entities(
self,
prisma_client: PrismaClient,
proxy_logging_obj: ProxyLogging,
transaction: _SpendTransaction,
updates: DBSpendUpdateTransactions,
) -> None:
await self._commit_spend_updates_to_db(
prisma_client, 0, proxy_logging_obj, updates, outer_transaction=transaction
)
async def _commit_spend_updates_to_db(
self,
prisma_client: PrismaClient,
@ -2178,6 +2305,7 @@ class DBSpendUpdateWriter:
proxy_logging_obj: ProxyLogging,
db_spend_update_transactions: DBSpendUpdateTransactions,
on_table_committed: Callable[[_SpendTableName], None] | None = None,
outer_transaction: _SpendTransaction | None = None,
):
"""
Commits all the spend `UPDATE` transactions to the Database
@ -2192,7 +2320,9 @@ class DBSpendUpdateWriter:
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with _spend_update_tx(prisma_client, "LiteLLM_UserTable") as transaction:
async with _spend_update_tx(
prisma_client, "LiteLLM_UserTable", outer_transaction=outer_transaction
) as transaction:
async with transaction.batch_() as batcher:
# Sort by ID for consistent lock ordering across pods to prevent deadlocks.
# batch_() issues statements sequentially within the tx, so iteration
@ -2204,6 +2334,8 @@ class DBSpendUpdateWriter:
)
break
except Exception as e:
if outer_transaction is not None:
raise
await self.handle_spend_update_failure(
e=e,
attempt=i,
@ -2223,6 +2355,7 @@ class DBSpendUpdateWriter:
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
end_user_list_transactions=end_user_list_transactions,
outer_transaction=outer_transaction,
)
if on_table_committed is not None:
on_table_committed("end_user_list_transactions")
@ -2233,7 +2366,9 @@ class DBSpendUpdateWriter:
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with _spend_update_tx(prisma_client, "LiteLLM_VerificationToken") as transaction:
async with _spend_update_tx(
prisma_client, "LiteLLM_VerificationToken", outer_transaction=outer_transaction
) as transaction:
async with transaction.batch_() as batcher:
# Sort by token for consistent lock ordering across pods to prevent deadlocks.
for token, response_cost in sorted(key_list_transactions.items()):
@ -2248,6 +2383,8 @@ class DBSpendUpdateWriter:
)
break
except Exception as e:
if outer_transaction is not None:
raise
await self.handle_spend_update_failure(
e=e,
attempt=i,
@ -2265,7 +2402,9 @@ class DBSpendUpdateWriter:
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with _spend_update_tx(prisma_client, "LiteLLM_TeamTable") as transaction:
async with _spend_update_tx(
prisma_client, "LiteLLM_TeamTable", outer_transaction=outer_transaction
) as transaction:
async with transaction.batch_() as batcher:
# Sort by team_id for consistent lock ordering across pods to prevent deadlocks.
for team_id, response_cost in sorted(team_list_transactions.items()):
@ -2278,6 +2417,8 @@ class DBSpendUpdateWriter:
)
break
except Exception as e:
if outer_transaction is not None:
raise
await self.handle_spend_update_failure(
e=e,
attempt=i,
@ -2303,11 +2444,15 @@ class DBSpendUpdateWriter:
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with _spend_update_tx(prisma_client, "LiteLLM_TeamMembership") as transaction:
async with _spend_update_tx(
prisma_client, "LiteLLM_TeamMembership", outer_transaction=outer_transaction
) as transaction:
await _write_team_member_spend(transaction, team_member_list_transactions)
# Transaction succeeded, break out of retry loop
break
except Exception as e:
if outer_transaction is not None:
raise
await self.handle_spend_update_failure(
e=e,
attempt=i,
@ -2320,7 +2465,7 @@ class DBSpendUpdateWriter:
# Invalidate cache for updated team memberships
# This ensures budget checks read fresh spend data from the database
if team_memberships_to_invalidate and proxy_logging_obj is not None:
if outer_transaction is None and team_memberships_to_invalidate and proxy_logging_obj is not None:
user_api_key_cache: Final = proxy_logging_obj.call_details.get("user_api_key_cache")
if user_api_key_cache is not None:
with service_target(AUTH_OBJECTS_TARGET):
@ -2340,7 +2485,9 @@ class DBSpendUpdateWriter:
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with _spend_update_tx(prisma_client, "LiteLLM_OrganizationTable") as transaction:
async with _spend_update_tx(
prisma_client, "LiteLLM_OrganizationTable", outer_transaction=outer_transaction
) as transaction:
async with transaction.batch_() as batcher:
# Sort by org_id for consistent lock ordering across pods to prevent deadlocks.
for org_id, response_cost in sorted(org_list_transactions.items()):
@ -2350,6 +2497,8 @@ class DBSpendUpdateWriter:
)
break
except Exception as e:
if outer_transaction is not None:
raise
await self.handle_spend_update_failure(
e=e,
attempt=i,
@ -2367,7 +2516,9 @@ class DBSpendUpdateWriter:
start_time = time.time()
try:
async with (
_spend_update_tx(prisma_client, "LiteLLM_OrganizationMembership") as transaction,
_spend_update_tx(
prisma_client, "LiteLLM_OrganizationMembership", outer_transaction=outer_transaction
) as transaction,
transaction.batch_() as batcher,
):
for key, response_cost in sorted(org_member_list_transactions.items()):
@ -2378,6 +2529,8 @@ class DBSpendUpdateWriter:
)
break
except Exception as e:
if outer_transaction is not None:
raise
await self.handle_spend_update_failure(
e=e,
attempt=i,
@ -2393,6 +2546,7 @@ class DBSpendUpdateWriter:
await DBSpendUpdateWriter._update_entity_spend_in_db(
entity_name="Project",
transactions=project_list_transactions,
outer_transaction=outer_transaction,
table_accessor="litellm_projecttable",
where_field="project_id",
n_retry_times=n_retry_times,
@ -2401,16 +2555,18 @@ class DBSpendUpdateWriter:
)
if on_table_committed is not None:
on_table_committed("project_list_transactions")
await DBSpendUpdateWriter._invalidate_project_caches(
project_ids=tuple(project_list_transactions or ()),
proxy_logging_obj=proxy_logging_obj,
)
if outer_transaction is None:
await DBSpendUpdateWriter._invalidate_project_caches(
project_ids=tuple(project_list_transactions or ()),
proxy_logging_obj=proxy_logging_obj,
)
### UPDATE TAG TABLE ###
tag_list_transactions: Final = db_spend_update_transactions["tag_list_transactions"]
await DBSpendUpdateWriter._update_entity_spend_in_db(
entity_name="Tag",
transactions=tag_list_transactions,
outer_transaction=outer_transaction,
table_accessor="litellm_tagtable",
where_field="tag_name",
n_retry_times=n_retry_times,
@ -2427,6 +2583,7 @@ class DBSpendUpdateWriter:
await DBSpendUpdateWriter._update_entity_spend_in_db(
entity_name="Model access group",
transactions=model_access_group_list_transactions,
outer_transaction=outer_transaction,
table_accessor="litellm_modelaccessgroupbudgettable",
where_field="access_group_name",
n_retry_times=n_retry_times,
@ -2441,6 +2598,7 @@ class DBSpendUpdateWriter:
await DBSpendUpdateWriter._update_entity_spend_in_db(
entity_name="Agent",
transactions=agent_list_transactions,
outer_transaction=outer_transaction,
table_accessor="litellm_agentstable",
where_field="agent_id",
n_retry_times=n_retry_times,
@ -2470,6 +2628,7 @@ class DBSpendUpdateWriter:
n_retry_times: int,
prisma_client: PrismaClient,
proxy_logging_obj: ProxyLogging,
outer_transaction: _SpendTransaction | None = None,
):
"""
Helper function to update spend for any entity type (team, org, tag, etc).
@ -2482,13 +2641,16 @@ class DBSpendUpdateWriter:
n_retry_times: Number of retries on failure
prisma_client: Prisma client instance
proxy_logging_obj: Proxy logging object
outer_transaction: Existing ledger transaction owning the complete delivery update
"""
verbose_proxy_logger.debug("%s Spend transactions: %s", entity_name, transactions)
if transactions is not None and len(transactions.keys()) > 0:
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with _spend_update_tx(prisma_client, _ENTITY_SPEND_MODELS[table_accessor]) as transaction:
async with _spend_update_tx(
prisma_client, _ENTITY_SPEND_MODELS[table_accessor], outer_transaction=outer_transaction
) as transaction:
async with transaction.batch_() as batcher:
# Sort by entity_id for consistent lock ordering across pods to prevent deadlocks.
for entity_id, response_cost in sorted(transactions.items()):
@ -2505,6 +2667,8 @@ class DBSpendUpdateWriter:
)
break
except Exception as e:
if outer_transaction is not None:
raise
await DBSpendUpdateWriter.handle_spend_update_failure(
e=e,
attempt=i,
@ -2838,6 +3002,12 @@ class DBSpendUpdateWriter:
entity_id_field="tag",
)
def _record_request_error_safely(self, *, payload: SpendLogsPayload, prisma_client: PrismaClient | None) -> None:
try:
self._record_request_error(payload=payload, prisma_client=prisma_client)
except Exception as error: # noqa: BLE001 # error rollup failure must not skip sibling writes
_report_spend_update_step_failure("_record_request_error", payload=payload, exc=error)
@staticmethod
def _record_request_error(*, payload: SpendLogsPayload, prisma_client: PrismaClient | None) -> None:
if prisma_client is None:

7
litellm/proxy/db/db_transaction_queue/tool_discovery_queue.py
View file

@ -55,3 +55,10 @@ class ToolDiscoveryQueue:
"""Return and clear all pending rows."""
items, self._pending = list(self._pending.values()), {}
return items
def requeue(self, items: Iterable[ToolDiscoveryQueueItem]) -> None:
"""Put rows from a failed write back, ahead of and merged with anything queued since the flush."""
since_flush: Final = self._pending
self._pending = {}
for item in (*items, *since_flush.values()):
self.add_update(item)

85
litellm/proxy/db/exception_handler.py
View file

@ -31,10 +31,29 @@ _CONNECTION_CAPACITY_PHRASES: Final = (
"too many connections for database",
"remaining connection slots are reserved",
)
_NOT_ACCEPTING_CONNECTIONS_SQLSTATE: Final = "57P03"
_NOT_ACCEPTING_CONNECTIONS_PHRASES: Final = (
"the database system is starting up",
"the database system is shutting down",
"the database system is not yet accepting connections",
"the database system is not accepting connections",
"the database system is in recovery mode",
)
_SERVER_TERMINATED_SQLSTATES: Final = frozenset({"57P01", "57P02", "57P05"})
_SERVER_TERMINATED_PHRASES: Final = (
"terminating connection due to administrator command",
"terminating connection due to immediate shutdown command",
"terminating connection because of unexpected sigquit signal",
"terminating connection because of crash of another server process",
"terminating connection due to idle-session timeout",
)
_UNREACHABLE_PRISMA_CODE: Final = "P1001"
_UNREACHABLE_PHRASES: Final = ("can't reach database server", "cannot reach database server")
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
_LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
_STATEMENT_TIMEOUT_SQLSTATE: Final = "57014"
_REJECTED_ROW_SQLSTATE_CLASSES: Final = frozenset({"22", "23", "54"})
_STATEMENT_TIMEOUT_PHRASE: Final = "canceling statement due to statement timeout"
@ -139,6 +158,8 @@ class PrismaDBExceptionHandler:
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return True
if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e):
return True
return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection
@staticmethod
@ -201,6 +222,20 @@ class PrismaDBExceptionHandler:
return type(e) is prisma.errors.DataError
@staticmethod
def is_rejected_row_error(e: Exception) -> bool:
"""True iff Postgres refused the rows themselves: a base ``DataError``, or a raw query failing with a data
exception (SQLSTATE class 22), integrity violation (class 23) or a value too large for an index (class 54). Systemic raw failures such as a missing
table (42P01) or a deadlock (40P01) stay False so the batch is retried instead of dropped."""
import prisma
if PrismaDBExceptionHandler.is_prisma_data_error(e):
return True
if not isinstance(e, _exception_types(prisma.errors.RawQueryError)):
return False
sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e)
return sqlstate is not None and sqlstate[:2] in _REJECTED_ROW_SQLSTATE_CLASSES
@staticmethod
def is_database_transport_error(e: Exception) -> bool:
"""
@ -289,6 +324,39 @@ class PrismaDBExceptionHandler:
error_message: Final = str(e).lower()
return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES)
@staticmethod
def is_database_not_accepting_connections_error(e: Exception) -> bool:
"""True iff Postgres refused the connection because it is starting up, shutting
down or in crash recovery (SQLSTATE 57P03). The refusal comes before any statement
runs, so nothing the write carried was applied."""
import prisma
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
if PrismaDBExceptionHandler.postgres_sqlstate(e) == _NOT_ACCEPTING_CONNECTIONS_SQLSTATE:
return True
error_message: Final = str(e).lower()
return any(phrase in error_message for phrase in _NOT_ACCEPTING_CONNECTIONS_PHRASES)
@staticmethod
def is_database_lifecycle_error(e: Exception) -> bool:
"""True iff Postgres failed the statement because of a server lifecycle event:
a refused connection during startup, shutdown or recovery (57P03), or a session
the server terminated (57P01, 57P02, 57P05). The server comes back on its own,
so the rows are not at fault. A terminated session may have committed before it
was cut, so only ``is_database_not_accepting_connections_error`` proves a
non-idempotent write never applied."""
import prisma
if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e):
return True
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
if PrismaDBExceptionHandler.postgres_sqlstate(e) in _SERVER_TERMINATED_SQLSTATES:
return True
error_message: Final = str(e).lower()
return any(phrase in error_message for phrase in _SERVER_TERMINATED_PHRASES)
@staticmethod
def is_lock_timeout_error(e: Exception) -> bool:
"""True iff Postgres cancelled a statement for exceeding ``lock_timeout``
@ -406,6 +474,8 @@ class PrismaDBExceptionHandler:
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return True
if PrismaDBExceptionHandler.is_database_lifecycle_error(e):
return True
if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e):
return True
if "cached plan must not change result type" in str(e).lower():
@ -680,3 +750,18 @@ def db_write_failure_is_requeue_safe(e: Exception) -> bool:
return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES)
sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e)
return sqlstate is None or sqlstate[:2] not in _DATA_REJECTED_SQLSTATE_CLASSES
def db_write_never_sent(e: Exception) -> bool:
"""Whether the database could not take the write at all, because it was unreachable (P1001),
refusing connections (57P03) or full (53300, P2024), so even a non-idempotent write can be sent again."""
if PrismaDBExceptionHandler.is_database_not_accepting_connections_error(e):
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return True
if _prisma_error_code(e) == _UNREACHABLE_PRISMA_CODE:
return True
if not PrismaDBExceptionHandler.is_prisma_error(e):
return False
error_message: Final = str(e).lower()
return any(phrase in error_message for phrase in _UNREACHABLE_PHRASES)

55
litellm/proxy/db/model_usage_rollup.py
View file

@ -32,14 +32,15 @@ class _UpsertTable(Protocol):
def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
class _ModelUsageBatch(Protocol):
litellm_dailymodelusage: _UpsertTable
class ModelUsageBatch(Protocol):
@property
def litellm_dailymodelusage(self) -> _UpsertTable: ...
def execute_raw(self, query: LiteralString, *args: object) -> None: ...
class _ModelUsageBatchManager(Protocol):
async def __aenter__(self) -> _ModelUsageBatch: ...
class ModelUsageBatchManager(Protocol):
async def __aenter__(self) -> ModelUsageBatch: ...
async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
@ -116,8 +117,8 @@ def build_model_usage_transaction(payload: SpendLogsPayload) -> ModelUsageTransa
)
def _model_usage_batch(prisma_client: PrismaClient) -> _ModelUsageBatchManager:
batch: Final[_ModelUsageBatchManager] = prisma_client.db.batch_()
def _model_usage_batch(prisma_client: PrismaClient) -> ModelUsageBatchManager:
batch: Final[ModelUsageBatchManager] = prisma_client.db.batch_()
return batch
@ -156,26 +157,16 @@ async def flush_model_usage_transactions(
"custom_llm_provider": key.custom_llm_provider,
"task_type": key.task_type,
}
batcher.litellm_dailymodelusage.upsert(
where={"date_model_group_model_custom_llm_provider_task_type": key_fields},
data={
"create": {
**key_fields,
"spend": spend,
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"request_count": len(requests),
"successful_requests": successful,
"failed_requests": failed,
},
"update": {
"spend": {"increment": spend},
"prompt_tokens": {"increment": prompt_tokens},
"completion_tokens": {"increment": completion_tokens},
"request_count": {"increment": len(requests)},
"successful_requests": {"increment": successful},
"failed_requests": {"increment": failed},
},
queue_model_usage_upsert(
batcher,
key_fields,
{
"spend": spend,
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"request_count": len(requests),
"successful_requests": successful,
"failed_requests": failed,
},
)
return
@ -183,3 +174,15 @@ async def flush_model_usage_transactions(
if attempt >= n_retry_times:
raise
await asyncio.sleep(2.0**attempt + random.uniform(0, 1))
def queue_model_usage_upsert(
batcher: ModelUsageBatch, key_fields: Mapping[str, str], increments: Mapping[str, int | float]
) -> None:
batcher.litellm_dailymodelusage.upsert(
where={"date_model_group_model_custom_llm_provider_task_type": key_fields},
data={
"create": {**key_fields, **increments},
"update": {name: {"increment": value} for name, value in increments.items()},
},
)

21
litellm/proxy/db/tool_registry_writer.py
View file

@ -23,7 +23,11 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
ToolDiscoveryQueue,
aggregate_tool_discoveries,
)
from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry, db_write_failure_is_requeue_safe
from litellm.proxy.db.exception_handler import (
call_with_db_reconnect_retry,
db_write_failure_is_requeue_safe,
db_write_never_sent,
)
from litellm.proxy.db.tool_table_bulk_upsert import build_tool_bulk_upsert
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
@ -134,8 +138,19 @@ async def flush_tool_discovery_queue(prisma_client: "PrismaClient", tool_discove
return
try:
await batch_upsert_tools(prisma_client=prisma_client, items=items)
except Exception as e: # noqa: BLE001 # a lost tool row only delays its discovery, the scheduler job must not fail
verbose_proxy_logger.debug("flush_tool_discovery_queue error (non-blocking): %s", e)
except Exception as e: # noqa: BLE001 # the rows go back to the queue or are dropped, the scheduler job must not fail
if db_write_never_sent(e):
tool_discovery_queue.requeue(items)
verbose_proxy_logger.warning(
"Tool discovery - %d tool rows went back to the queue for the next tick. Error: %s", len(items), str(e)
)
return
verbose_proxy_logger.error(
"Tool discovery - dropped %d tool rows: the failed statement may have applied or the database "
"refused the data, so re-sending it is not safe. Error: %s",
len(items),
str(e),
)
async def commit_tool_discoveries(

61
litellm/proxy/guardrails/usage_tracking.py
View file

@ -11,7 +11,7 @@ from datetime import datetime, timezone
from functools import partial
from itertools import groupby
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, TypedDict
@ -22,6 +22,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import billed_guard
from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
from litellm.proxy.db.spend_log_batching import spend_log_write_batches
from litellm.proxy.utils import PrismaClient
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import (
DailyGuardrailMetricsRepository,
DailyGuardrailUsageUnitsRepository,
@ -40,7 +41,7 @@ _RowKey = TypeVar("_RowKey")
_RowValue = TypeVar("_RowValue")
class _UsageUnitKey(NamedTuple):
class UsageUnitKey(NamedTuple):
guardrail_id: str
date: str
team_id: str
@ -48,7 +49,7 @@ class _UsageUnitKey(NamedTuple):
usage_unit: str
class _UsageUnitIncrement(NamedTuple):
class UsageUnitIncrement(NamedTuple):
units: int
cost: float
"""USD for the priced share of units."""
@ -56,14 +57,14 @@ class _UsageUnitIncrement(NamedTuple):
"""Units recorded with no known price, the share cost leaves out."""
def _usage_unit_key(entry: tuple[_UsageUnitKey, _UsageUnitIncrement]) -> _UsageUnitKey:
def _usage_unit_key(entry: tuple[UsageUnitKey, UsageUnitIncrement]) -> UsageUnitKey:
return entry[0]
def _usage_unit_increment(units: int, cost: float | None) -> _UsageUnitIncrement:
def _usage_unit_increment(units: int, cost: float | None) -> UsageUnitIncrement:
if cost is None:
return _UsageUnitIncrement(units=units, cost=0.0, untracked_units=units)
return _UsageUnitIncrement(units=units, cost=cost, untracked_units=0)
return UsageUnitIncrement(units=units, cost=0.0, untracked_units=units)
return UsageUnitIncrement(units=units, cost=cost, untracked_units=0)
class _MetricsKey(NamedTuple):
@ -89,22 +90,22 @@ class PendingRollups:
def __init__(self) -> None:
self.lock: Final = asyncio.Lock()
self.metrics: Mapping[_MetricsKey, Mapping[str, int]] = MappingProxyType({})
self.units: Mapping[_UsageUnitKey, _UsageUnitIncrement] = MappingProxyType({})
self.units: Mapping[UsageUnitKey, UsageUnitIncrement] = MappingProxyType({})
_PENDING_ROLLUPS: Final = PendingRollups()
_NO_COUNTERS: Final[Mapping[str, int]] = MappingProxyType({})
_NO_INCREMENT: Final = _UsageUnitIncrement(units=0, cost=0.0, untracked_units=0)
_NO_INCREMENT: Final = UsageUnitIncrement(units=0, cost=0.0, untracked_units=0)
def _merged_keys(base: Mapping[_RowKey, object], extra: Mapping[_RowKey, object]) -> tuple[_RowKey, ...]:
return (*base, *(key for key in extra if key not in base))
def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitIncrement:
def _summed_increments(increments: Iterable[UsageUnitIncrement]) -> UsageUnitIncrement:
materialized: Final = tuple(increments)
return _UsageUnitIncrement(
return UsageUnitIncrement(
units=sum(i.units for i in materialized),
cost=sum(i.cost for i in materialized),
untracked_units=sum(i.untracked_units for i in materialized),
@ -112,8 +113,8 @@ def _summed_increments(increments: Iterable[_UsageUnitIncrement]) -> _UsageUnitI
def _merged_unit_rows(
base: Mapping[_UsageUnitKey, _UsageUnitIncrement], extra: Mapping[_UsageUnitKey, _UsageUnitIncrement]
) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]:
base: Mapping[UsageUnitKey, UsageUnitIncrement], extra: Mapping[UsageUnitKey, UsageUnitIncrement]
) -> Mapping[UsageUnitKey, UsageUnitIncrement]:
return MappingProxyType(
{
key: _summed_increments((base.get(key, _NO_INCREMENT), extra.get(key, _NO_INCREMENT)))
@ -250,7 +251,7 @@ def _parse_payload_start_time(payload: Mapping[str, object]) -> datetime | None:
def _iter_usage_unit_increments(
logs_to_process: Sequence[Mapping[str, object]],
) -> Iterator[tuple[_UsageUnitKey, _UsageUnitIncrement]]:
) -> Iterator[tuple[UsageUnitKey, UsageUnitIncrement]]:
for payload in logs_to_process:
start_time = _parse_payload_start_time(payload)
if not payload.get("request_id") or start_time is None:
@ -266,14 +267,14 @@ def _iter_usage_unit_increments(
cost_by_unit = billed_guardrail_cost_by_unit(entry)
for unit_name, units in usage.items():
if isinstance(units, int) and not isinstance(units, bool) and units > 0:
key = _UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name))
key = UsageUnitKey(guardrail_id, date_key, team_id, api_key, str(unit_name))
cost = cost_by_unit.get(str(unit_name)) if cost_by_unit is not None else None
yield key, _usage_unit_increment(units=units, cost=cost)
def _sum_usage_unit_increments(
logs_to_process: Sequence[Mapping[str, object]],
) -> Mapping[_UsageUnitKey, _UsageUnitIncrement]:
) -> Mapping[UsageUnitKey, UsageUnitIncrement]:
ordered: Final = sorted(_iter_usage_unit_increments(logs_to_process), key=_usage_unit_key)
return MappingProxyType(
{
@ -283,8 +284,16 @@ def _sum_usage_unit_increments(
)
async def _upsert_usage_unit_row(
prisma_client: PrismaClient, key: _UsageUnitKey, increment: _UsageUnitIncrement
class GuardrailUsageDatabase(Protocol):
@property
def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ...
async def upsert_usage_unit_row(
prisma_client: PrismaClient,
key: UsageUnitKey,
increment: UsageUnitIncrement,
outer_transaction: GuardrailUsageDatabase | None = None,
) -> None:
row: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsCreateInput] = {
"guardrail_id": key.guardrail_id,
@ -314,7 +323,12 @@ async def _upsert_usage_unit_row(
"untracked_units": {"increment": increment.untracked_units},
},
}
await DailyGuardrailUsageUnitsRepository(prisma_client).table.upsert(where=where, data=data)
table: Final = (
outer_transaction.litellm_dailyguardrailusageunits
if outer_transaction is not None
else DailyGuardrailUsageUnitsRepository(prisma_client).table
)
await table.upsert(where=where, data=data)
async def _upsert_metrics_row(prisma_client: PrismaClient, key: _MetricsKey, agg: Mapping[str, int]) -> None:
@ -363,7 +377,7 @@ async def process_spend_logs_guardrail_usage(
)
index_rows_by_key: Final[dict[tuple[str, str], dict[str, object]]] = {}
for payload in logs_to_process:
for payload in (row for row in logs_to_process if row.get("accounting_log_disabled") is not True):
request_id = payload.get("request_id")
start_time = _parse_payload_start_time(payload)
if not isinstance(request_id, str) or not request_id or start_time is None:
@ -416,7 +430,10 @@ async def process_spend_logs_guardrail_usage(
{key: agg for key, agg in daily_guardrail.items() if int(agg["requests_evaluated"]) > 0}
)
metrics_rows: Final = _merged_metric_rows(pending_metrics, evaluated_metrics)
unit_rows: Final = _merged_unit_rows(pending_units, _sum_usage_unit_increments(logs_to_process))
unit_rows: Final = _merged_unit_rows(
pending_units,
_sum_usage_unit_increments(tuple(row for row in logs_to_process if "accounting_payload" not in row)),
)
if not metrics_rows and not index_rows and not unit_rows:
return
@ -435,7 +452,7 @@ async def process_spend_logs_guardrail_usage(
metrics_rows, partial(_upsert_metrics_row, prisma_client), "daily metrics", sleep
)
failed_units: Final = await _upsert_rows_with_retry(
unit_rows, partial(_upsert_usage_unit_row, prisma_client), "usage unit", sleep
unit_rows, partial(upsert_usage_unit_row, prisma_client), "usage unit", sleep
)
if failed_metrics or failed_units:
async with pending.lock:

20
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -9,6 +9,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast
from pydantic import TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
@ -38,6 +39,7 @@ from litellm.proxy.auth.auth_checks import (
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
from litellm.proxy.db.db_spend_update_writer import (
ACCOUNTING_LEDGER_KWARG,
DBSpendUpdateWriter,
debitable_model_access_groups,
get_llm_router,
@ -657,6 +659,7 @@ class _ProxyDBLogger(CustomLogger):
token: Final = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict)
end_user_id: Final = None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id
accounting_rows: Final = tuple((cost, dict(row)) for cost, row in spend_rows)
charged: Final = [
await self._spend_writer().update_database(
token=token,
@ -664,17 +667,17 @@ class _ProxyDBLogger(CustomLogger):
user_id=user_api_key_dict.user_id,
end_user_id=end_user_id,
team_id=user_api_key_dict.team_id,
kwargs=dict(row_kwargs),
kwargs=row_kwargs,
completion_response=original_exception,
start_time=start_time,
end_time=datetime.now(),
org_id=user_api_key_dict.org_id,
project_id=user_api_key_dict.project_id,
)
for row_cost, row_kwargs in spend_rows
for row_cost, row_kwargs in accounting_rows
]
counted_rows: Final = tuple(
row for row, row_charged in zip(spend_rows, charged, strict=True) if row_charged and row[0] > 0
row for row, row_charged in zip(accounting_rows, charged, strict=True) if row_charged and row[0] > 0
)
if counted_rows and not ProxyUpdateSpend.disable_spend_updates():
await self._increment_failure_spend_counters(
@ -753,6 +756,7 @@ class _ProxyDBLogger(CustomLogger):
router=get_llm_router(),
),
project_id=user_api_key_dict.project_id,
**_ledger_counter_options(row_kwargs.get(ACCOUNTING_LEDGER_KWARG)),
)
except Exception: # noqa: BLE001 # one row's counter failure must not keep the other rows' counters from landing
verbose_proxy_logger.exception("Failed to raise spend counters for a failure spend row")
@ -1196,6 +1200,14 @@ def _get_request_tags_for_cost_tracking(
return None
class _LedgerCounterOptions(TypedDict):
windows_from_ledger: ReadOnly[NotRequired[bool]]
def _ledger_counter_options(ledger: object) -> _LedgerCounterOptions:
return {"windows_from_ledger": True} if ledger is True else {}
class _ChargeSpend(Protocol):
async def __call__(
self,
@ -1256,6 +1268,7 @@ class _IncrementSpendCounters(Protocol):
request_started_at: datetime | None = None,
model_access_groups: Sequence[str] | None = None,
project_id: str | None = None,
windows_from_ledger: bool = False,
) -> None: ...
@ -1393,6 +1406,7 @@ async def _update_database_and_spend_counters_in_batch(
request_started_at=start_time,
model_access_groups=model_access_groups,
project_id=project_id,
**_ledger_counter_options(_BATCH_CALLBACK_FIELDS.validate_python(kwargs).get(ACCOUNTING_LEDGER_KWARG)),
)
except Exception:
if budget_reservation is not None:

85
litellm/proxy/proxy_server.py
View file

@ -6,6 +6,7 @@ import importlib
import inspect
import io
import itertools
import logging
import os
import random
import re
@ -424,9 +425,13 @@ from litellm.messages.dispatch import messages_target
from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot
from litellm.proxy._types import *
from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling
from litellm.proxy.alerting_endpoints.model_deprecation_alerts import (
router as model_deprecation_alerts_router,
)
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
router as analytics_router,
)
from litellm.proxy.anthropic_endpoints.answered_status import is_provider_overload
from litellm.proxy.auth.auth_checks import (
ROLE_BASED_PERMISSIONS_ADAPTER,
ExperimentalUIJWTToken,
@ -3623,6 +3628,7 @@ async def increment_spend_counters(
request_started_at: datetime | None = None,
model_access_groups: Sequence[str] | None = None,
project_id: str | None = None,
windows_from_ledger: bool = False,
):
"""
Atomically increment spend counters for budget enforcement.
@ -3659,6 +3665,7 @@ async def increment_spend_counters(
request_started_at=request_started_at,
model_access_groups=model_access_groups,
project_id=project_id,
windows_from_ledger=windows_from_ledger,
)
@ -3688,6 +3695,50 @@ def _finish_pending_read(absent_key: str, pending_read: str, remember_absent: bo
user_api_key_cache.budget_windows_absent_cache.delete_cache(absent_key)
async def accounting_window_keys(token: str | None, team_id: str | None) -> tuple[Mapping[str, object], ...]:
scopes: Final = tuple(
(entity_type, entity_id, cache_key)
for entity_type, entity_id, cache_key in (
(Litellm_EntityType.KEY, token, token),
(Litellm_EntityType.TEAM, team_id, f"team_id:{team_id}"),
)
if entity_id is not None and cache_key is not None
)
snapshots: Final = tuple(
[
await _accounting_scope_windows(entity_type, entity_id, cache_key)
for entity_type, entity_id, cache_key in scopes
]
)
from itertools import chain
return tuple(chain.from_iterable(snapshots))
async def _accounting_scope_windows(
entity_type: Litellm_EntityType, entity_id: str, cache_key: str
) -> tuple[Mapping[str, object], ...]:
from litellm.proxy.spend_tracking.budget_reservation import fixed_budget_window
cached: Final[object] = _UNTYPED_VALUE.validate_python(await user_api_key_cache.async_get_cache(key=cache_key))
limits: Final = (
await _stored_budget_limits(entity_type, entity_id) if cached is None else _cached_budget_limits(cached)
)
if not isinstance(limits, list):
return ()
windows: Final = TypeAdapter(tuple[object, ...]).validate_python(limits)
return tuple(
{
"entity_type": entity_type.value,
"entity_id": entity_id,
"window_duration": fixed[0],
"window_start": fixed[1].isoformat(),
}
for window in windows
if (fixed := fixed_budget_window(window)) is not None
)
async def _stored_budget_limits(entity_type: Litellm_EntityType, entity_id: str) -> object:
"""Read only: caching the row here could race a delete and put a just-deleted key back into the auth cache."""
if prisma_client is None:
@ -3729,6 +3780,7 @@ async def _increment_spend_counters_batched(
request_started_at: datetime | None,
model_access_groups: Sequence[str] | None,
project_id: str | None = None,
windows_from_ledger: bool = False,
):
"""Runs inside one spend counter batch: the reservation reconcile and the warm checks share a single MGET, and
the reconcile adjustments go out in the same INCRBYFLOAT pipeline as the counter increments."""
@ -3770,15 +3822,16 @@ async def _increment_spend_counters_batched(
if window_counter not in reserved_counter_keys
else None
)
await _enqueue_window_spend_row_update(
entity_type=entity_type,
entity_id=entity_id,
reset_at=window_reset_at,
window_duration=duration,
window_start=window_start,
increment=cost,
request_started_at=request_started_at,
)
if not windows_from_ledger:
await _enqueue_window_spend_row_update(
entity_type=entity_type,
entity_id=entity_id,
reset_at=window_reset_at,
window_duration=duration,
window_start=window_start,
increment=cost,
request_started_at=request_started_at,
)
return pending_window
async def _window_increments(
@ -13396,6 +13449,13 @@ async def completion(
_response.choices[0].text = e.message
return _response
except Exception as e:
if is_provider_overload(e):
raise await base_llm_response_processor.handle_llm_api_exception(
e=e,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
@ -18626,7 +18686,11 @@ async def model_deprecations(
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
```
"""
return collect_model_deprecations(llm_router=llm_router, warn_within_days=warn_within_days)
return collect_model_deprecations(
llm_router=llm_router,
warn_within_days=warn_within_days,
successor_log_level=logging.DEBUG,
)
def get_model_group_info(
@ -22393,6 +22457,7 @@ app.include_router(prompt_caching_requests_router)
app.include_router(router_settings_router)
app.include_router(fallback_management_router)
app.include_router(cache_settings_router)
app.include_router(model_deprecation_alerts_router)
app.include_router(coordination_redis_settings_router)
app.include_router(user_agent_analytics_router)
app.include_router(gateway_request_router)

17
litellm/proxy/schema.prisma
View file

@ -2141,6 +2141,23 @@ model LiteLLM_AccountingDelivery {
@@index([target, state, next_attempt_at])
}
model LiteLLM_SpendLogDelivery {
delivery_id String @id @default(dbgenerated("gen_random_uuid()::text"))
request_id String
state AccountingDeliveryState @default(pending)
accounting_payload Json
is_batch Boolean
attempts Int @default(0)
next_attempt_at DateTime @default(now())
applied_at DateTime?
last_error String?
created_at DateTime @default(now())
@@index([state, next_attempt_at])
@@index([state, applied_at])
@@index([request_id])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String

9
litellm/proxy/spend_tracking/budget_reservation.py
View file

@ -1264,6 +1264,15 @@ def _get_entry_reserved_cost(entry: dict, default_reserved_cost: float) -> float
return default_reserved_cost
def fixed_budget_window(window: object) -> tuple[str, datetime] | None:
fields: Final = _coerce_window(window)
start: Final = get_budget_window_start(window)
duration: Final = fields.get("budget_duration")
if not fields.get("reset_at") or not isinstance(duration, str) or start is None:
return None
return duration, start
def get_budget_window_start(window: object) -> datetime | None:
window_dict: Final = _coerce_window(window)
budget_duration: Final = window_dict.get("budget_duration")

10
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -161,9 +161,7 @@ def _is_non_secret_key_value(value: str, *, key_alias: object = None) -> bool:
)
def _redact_logged_api_key(
value: str | None, *, already_redacted: bool = False, key_alias: object = None
) -> str | None:
def redact_logged_api_key(value: str | None, *, already_redacted: bool = False, key_alias: object = None) -> str | None:
if not isinstance(value, str) or not value:
return None
stripped: Final = re.sub(r"(?i)^bearer ", "", value)
@ -327,7 +325,7 @@ def _get_spend_logs_metadata(
and _is_non_secret_key_value(_trusted_hash, key_alias=_key_alias)
and _trusted_hash == _raw_key
)
clean_metadata["user_api_key"] = _redact_logged_api_key(
clean_metadata["user_api_key"] = redact_logged_api_key(
_raw_key, already_redacted=_already_redacted, key_alias=_key_alias
)
clean_metadata["applied_guardrails"] = applied_guardrails
@ -692,14 +690,14 @@ def get_logging_payload(
and _is_non_secret_key_value(_trusted_hash, key_alias=_key_alias)
and _trusted_hash == api_key
)
api_key = _redact_logged_api_key(api_key, already_redacted=_key_already_redacted, key_alias=_key_alias) or ""
api_key = redact_logged_api_key(api_key, already_redacted=_key_already_redacted, key_alias=_key_alias) or ""
if (
standard_logging_payload is not None
): # [TODO] migrate completely to sl payload. currently missing pass-through endpoint data
api_key = (
api_key
or _redact_logged_api_key(
or redact_logged_api_key(
standard_logging_payload["metadata"].get("user_api_key_hash"),
already_redacted=True,
key_alias=standard_logging_payload["metadata"].get("user_api_key_alias"),

123
litellm/proxy/utils.py
View file

@ -215,6 +215,7 @@ from litellm.proxy.db.db_url_settings import (
from litellm.proxy.db.exception_handler import (
PrismaDBExceptionHandler,
call_with_db_reconnect_retry,
db_write_never_sent,
)
from litellm.proxy.db.health_check_latest import (
LatestHealthCheckRow,
@ -279,6 +280,7 @@ from litellm.proxy.policy_engine.policy_registry import get_policy_registry
from litellm.proxy.policy_engine.policy_resolver import PolicyResolver
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.config_repository import ConfigRepository
from litellm.repositories.spend_log_insert import RejectedSpendLogRow
from litellm.repositories.table_repositories import (
EndUserRepository,
HealthCheckRepository,
@ -8446,14 +8448,17 @@ async def enqueue_spend_logs(
"""
added: Final = sum(spend_log_row_bytes(row) for row in logs)
async with prisma_client._spend_log_transactions_lock:
queued: Final = (
queued: Final[tuple[Mapping[str, object], ...]] = (
tuple(logs) + tuple(prisma_client.spend_log_transactions)
if at_head
else tuple(prisma_client.spend_log_transactions) + tuple(logs)
)
kept, kept_bytes = spend_log_queue_within_budget(queued, PrismaClient.spend_log_queue_bytes + added, max_bytes)
prisma_client.spend_log_transactions[:] = kept
PrismaClient.spend_log_queue_bytes = kept_bytes
rescued: Final = tuple(
accounting_only_spend_log(row) for row in queued[: len(queued) - len(kept)] if _carries_a_charge(row)
)
prisma_client.spend_log_transactions[:] = (*rescued, *kept)
PrismaClient.spend_log_queue_bytes = kept_bytes + sum(spend_log_row_bytes(row) for row in rescued)
if len(kept) < len(queued):
verbose_proxy_logger.error(
"Spend tracking - spend log queue is at its %d byte budget; dropped the %d oldest spend logs",
@ -8508,13 +8513,18 @@ class ProxyUpdateSpend:
prisma_client: PrismaClient,
proxy_logging_obj: ProxyLogging,
end_user_list_transactions: dict[str, float],
outer_transaction: _EndUserSpendTransaction | None = None,
):
for i in range(n_retry_times + 1):
start_time = time.time()
try:
async with (
db_span("update_end_user_spend", "LiteLLM_EndUserTable"),
_end_user_spend_transaction(prisma_client.db, timedelta(seconds=60)) as transaction,
(
contextlib.nullcontext(outer_transaction)
if outer_transaction is not None
else _end_user_spend_transaction(prisma_client.db, timedelta(seconds=60))
) as transaction,
):
batcher: _EndUserSpendBatch
async with transaction.batch_() as batcher:
@ -8536,6 +8546,8 @@ class ProxyUpdateSpend:
break
except Exception as e:
if outer_transaction is not None:
raise
await DBSpendUpdateWriter.handle_spend_update_failure(
e=e,
attempt=i,
@ -8575,11 +8587,18 @@ class ProxyUpdateSpend:
for i in range(n_retry_times + 1):
try:
base_url = os.getenv("SPEND_LOGS_URL", None)
if len(logs_to_process) > 0 and base_url is not None and db_writer_client is not None:
await _commit_export_accounting(
prisma_client, rows_to_write, base_url if db_writer_client is not None else None
)
if (
base_url is not None
and db_writer_client is not None
and any(row.get("accounting_log_disabled") is not True for row in rows_to_write)
):
if not base_url.endswith("/"):
base_url += "/"
verbose_proxy_logger.debug("base_url: %s", base_url)
json_data = json.dumps(rows_to_write)
json_data = json.dumps(_spend_log_export_rows(rows_to_write))
response = await db_writer_client.post(
url=base_url + "spend/update",
data=json_data,
@ -8657,6 +8676,28 @@ class ProxyUpdateSpend:
return False
def _spend_log_export_rows(rows: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
return tuple(
{
key: value
for key, value in row.items()
if key not in {"accounting_payload", "accounting_log_disabled", "accounting_delivery_id"}
}
for row in rows
if row.get("accounting_log_disabled") is not True
)
async def _commit_export_accounting(
prisma_client: PrismaClient, rows: Sequence[Mapping[str, object]], base_url: str | None
) -> None:
if base_url is None:
return
await SpendLogsRepository(prisma_client).insert_with_deliveries(
tuple(row for row in rows if "accounting_payload" in row), write_spend_logs=False
)
async def commit_tool_discoveries(prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging) -> None:
"""Scheduler job writing discovered tools to LiteLLM_ToolTable, on its own lock, next to update_spend."""
writer: Final = proxy_logging_obj.db_spend_update_writer
@ -8832,9 +8873,20 @@ async def update_daily_tag_spend(
def _rollup_batch_never_applied(e: Exception) -> bool:
"""True when a drained rollup batch provably never reached its rows: Postgres had no
connection for it (53300) or cancelled it under ``lock_timeout`` before it took the row
lock (55P03). Either way re-sending the batch cannot double-count, so it is requeued."""
return PrismaDBExceptionHandler.is_database_capacity_error(e) or PrismaDBExceptionHandler.is_lock_timeout_error(e)
connection for it (53300), was unreachable or refusing connections (P1001, 57P03), or
cancelled it under ``lock_timeout`` before it took the row lock (55P03). In each case
re-sending the batch cannot double-count, so it is requeued."""
return (
PrismaDBExceptionHandler.is_database_capacity_error(e)
or db_write_never_sent(e)
or PrismaDBExceptionHandler.is_lock_timeout_error(e)
)
def _rollup_failure_stops_job(e: Exception) -> bool:
"""True when the database cannot take any write, so the job stops instead of draining
the queues after this one into the same failure."""
return PrismaDBExceptionHandler.is_database_capacity_error(e) or db_write_never_sent(e)
async def update_spend_logs_job(
@ -8925,12 +8977,12 @@ async def _run_spend_logs_job(
*prisma_client.tool_usage_transactions,
]
verbose_proxy_logger.warning(
"Spend tracking - database out of connections or rollup row lock timed out during tool usage "
"Spend tracking - database unavailable or rollup row lock timed out during tool usage "
"flush, requeued %s tool usage transactions for the next flush: %s",
len(tool_usage_to_process),
tool_tracking_err,
)
if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err):
if _rollup_failure_stops_job(tool_tracking_err):
raise
else:
verbose_proxy_logger.error(
@ -8957,12 +9009,12 @@ async def _run_spend_logs_job(
*prisma_client.model_usage_transactions,
]
verbose_proxy_logger.warning(
"Spend tracking - database out of connections or rollup row lock timed out during model usage "
"Spend tracking - database unavailable or rollup row lock timed out during model usage "
"flush, requeued %s model usage transactions for the next flush: %s",
len(model_usage_to_process),
model_usage_err,
)
if PrismaDBExceptionHandler.is_database_capacity_error(model_usage_err):
if _rollup_failure_stops_job(model_usage_err):
raise
else:
verbose_proxy_logger.error(
@ -9139,10 +9191,45 @@ def _is_transient_spend_log_write_error(e: Exception) -> bool:
return (
PrismaDBExceptionHandler.is_database_transport_error(e)
or PrismaDBExceptionHandler.is_prisma_engine_internal_error(e)
or PrismaDBExceptionHandler.is_database_lifecycle_error(e)
or PrismaDBExceptionHandler.is_deadlock_error(e)
)
_ACCOUNTING_ONLY_KEYS: Final = frozenset({"request_id", "call_type", "accounting_payload", "accounting_delivery_id"})
def _carries_a_charge(row: Mapping[str, object]) -> bool:
return "accounting_payload" in row
def accounting_only_spend_log(row: Mapping[str, object]) -> Mapping[str, object]:
return {
**{key: value for key, value in row.items() if key in _ACCOUNTING_ONLY_KEYS},
"accounting_log_disabled": True,
}
async def _keep_charges_of_dropped_logs(repo: SpendLogsRepository, rows: Sequence[Mapping[str, object]]) -> None:
charged: Final = tuple(accounting_only_spend_log(row) for row in rows if _carries_a_charge(row))
if not charged:
return
try:
await repo.insert_with_deliveries(charged)
except Exception as e: # noqa: BLE001 # the request log is already lost; keep the flush going for other rows
if not (
PrismaDBExceptionHandler.is_rejected_row_error(e)
or isinstance(e, (UnicodeEncodeError, RejectedSpendLogRow))
):
raise
spend_log_error(
"Spend tracking - could not keep the charges of %d dropped spend log rows. error=%s",
len(charged),
str(e),
exc=e,
)
async def _create_spend_logs_with_poison_isolation(
repo: SpendLogsRepository,
rows: Sequence[Mapping[str, object]],
@ -9171,10 +9258,13 @@ async def _create_spend_logs_with_poison_isolation(
persist. Returns the budget left after this subtree.
"""
try:
await repo.table.create_many(data=rows, skip_duplicates=True)
await repo.insert_with_deliveries(rows)
return failure_budget
except Exception as e:
if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)):
if not (
PrismaDBExceptionHandler.is_rejected_row_error(e)
or isinstance(e, (UnicodeEncodeError, RejectedSpendLogRow))
):
raise
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
raise
@ -9189,6 +9279,7 @@ async def _create_spend_logs_with_poison_isolation(
str(e),
exc=e,
)
await _keep_charges_of_dropped_logs(repo, rows)
return budget_left
if budget_left <= 0:
spend_log_error(
@ -9196,6 +9287,7 @@ async def _create_spend_logs_with_poison_isolation(
"isolation failure budget exhausted for this flush",
len(rows),
)
await _keep_charges_of_dropped_logs(repo, rows)
return 0
mid: Final = len(rows) // 2
remaining: Final = await _create_spend_logs_with_poison_isolation(repo, rows[:mid], budget_left)
@ -9205,6 +9297,7 @@ async def _create_spend_logs_with_poison_isolation(
"isolation failure budget exhausted for this flush",
len(rows) - mid,
)
await _keep_charges_of_dropped_logs(repo, rows[mid:])
return 0
return await _create_spend_logs_with_poison_isolation(repo, rows[mid:], remaining)

193
litellm/repositories/accounting_delivery_repository.py Normal file
View file

@ -0,0 +1,193 @@
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import timedelta
from itertools import chain
from types import TracebackType
from typing import TYPE_CHECKING, Final, Literal, Protocol
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from litellm.repositories.prisma_protocols import TableActions
if TYPE_CHECKING:
from litellm.proxy.db.db_spend_update_writer import SpendBatchManager
class LedgerTransaction(Protocol):
@property
def litellm_dailyguardrailusageunits(self) -> TableActions[object]: ...
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
async def execute_raw(self, query: str, *args: object) -> int: ...
def batch_(self) -> "SpendBatchManager": ...
class _TransactionManager(Protocol):
async def __aenter__(self) -> LedgerTransaction: ...
async def __aexit__(
self, exc_type: type[BaseException] | None, exc: BaseException | None, exc_tb: TracebackType | None
) -> None: ...
class LedgerDatabase(Protocol):
def tx(self, *, timeout: timedelta) -> _TransactionManager: ...
class _Delivery(BaseModel):
model_config = ConfigDict(frozen=True)
delivery_id: str
attempts: int
accounting_payload: JsonValue
@dataclass(frozen=True, slots=True)
class DeliveryProjection:
row: Mapping[str, object]
@dataclass(frozen=True, slots=True)
class DeliveryBatchResult:
applied: int = 0
failed: int = 0
parked: tuple[str, ...] = ()
error: str | None = None
class DeliveryMetrics(BaseModel):
model_config = ConfigDict(frozen=True)
pending_count: int
parked_count: int
oldest_pending_seconds: float
batch_count: int
table_bytes: int
_DELIVERIES: Final = TypeAdapter(tuple[_Delivery, ...])
_PAYLOAD: Final = TypeAdapter(dict[str, object])
_TIMEOUT: Final = timedelta(seconds=60)
_CLAIM: Final = """SELECT delivery_id, attempts, accounting_payload FROM "LiteLLM_SpendLogDelivery"
WHERE state = 'pending' AND next_attempt_at <= CURRENT_TIMESTAMP
AND (attempts > 0) = $2::boolean
ORDER BY next_attempt_at, delivery_id LIMIT $1::integer FOR UPDATE SKIP LOCKED"""
_APPLIED: Final = """UPDATE "LiteLLM_SpendLogDelivery"
SET state = 'applied', applied_at = CURRENT_TIMESTAMP, last_error = NULL
WHERE delivery_id = ANY($1::text[]) AND state = 'pending' """
_FAILED: Final = """UPDATE "LiteLLM_SpendLogDelivery"
SET attempts = attempts + 1, last_error = $3,
next_attempt_at = CURRENT_TIMESTAMP + make_interval(secs => LEAST(300, power(2, LEAST(attempts, 8)))::integer),
state = $4::"AccountingDeliveryState"
WHERE delivery_id = $1 AND state = 'pending' AND attempts = $2::integer
RETURNING delivery_id, attempts, accounting_payload"""
def _is_permanent_failure(error: Exception) -> bool:
"""A delivery whose own data failed fails the same way on every retry, while any other error is the database's
and clears once it recovers."""
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
if PrismaDBExceptionHandler.is_database_service_unavailable_error(error):
return False
return isinstance(error, (ValueError, TypeError, KeyError)) or PrismaDBExceptionHandler.is_rejected_row_error(error)
class AccountingDeliveryRepository:
def __init__(self, database: LedgerDatabase) -> None:
self._database = database
async def metrics(self) -> DeliveryMetrics:
async with self._database.tx(timeout=_TIMEOUT) as transaction:
rows: Final = await transaction.query_raw(
"SELECT COUNT(*) FILTER (WHERE state = 'pending') AS pending_count, "
"COUNT(*) FILTER (WHERE state = 'parked') AS parked_count, "
"COALESCE(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - MIN(created_at) FILTER (WHERE state = 'pending')), 0) "
"AS oldest_pending_seconds, COUNT(*) FILTER (WHERE is_batch) AS batch_count, "
"pg_total_relation_size('\"LiteLLM_SpendLogDelivery\"') AS table_bytes "
'FROM "LiteLLM_SpendLogDelivery"'
)
return DeliveryMetrics.model_validate(rows[0])
async def cleanup_applied(self, *, retention_days: int = 1, batch_size: int = 1000) -> int:
if retention_days < 1 or batch_size < 1:
raise ValueError("Delivery retention and cleanup batch size must be positive")
async with self._database.tx(timeout=_TIMEOUT) as transaction:
return await transaction.execute_raw(
'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id IN ('
'SELECT delivery_id FROM "LiteLLM_SpendLogDelivery" '
"WHERE state = 'applied' AND NOT is_batch AND applied_at < CURRENT_TIMESTAMP - make_interval(days => $1::integer) "
"ORDER BY applied_at, delivery_id LIMIT $2::integer FOR UPDATE SKIP LOCKED)",
retention_days,
batch_size,
)
async def apply_pending(
self,
apply: Callable[[LedgerTransaction, tuple[DeliveryProjection, ...]], Awaitable[None]],
*,
batch_size: int = 100,
max_attempts: int = 5,
retry_only: bool | None = None,
) -> DeliveryBatchResult:
if batch_size < 1 or max_attempts < 1:
raise ValueError("Delivery batch size and maximum attempts must be positive")
claimed: tuple[_Delivery, ...] = () # rebind-ok: preserve claims for retry accounting after rollback
try:
async with self._database.tx(timeout=_TIMEOUT) as transaction:
retries: Final = (
_DELIVERIES.validate_python(await transaction.query_raw(_CLAIM, 1, True))
if retry_only is not False
else ()
)
claimed = retries or ( # rebind-ok: retain claimed identities after rollback
_DELIVERIES.validate_python(await transaction.query_raw(_CLAIM, batch_size, False))
if retry_only is not True
else ()
)
if not claimed:
return DeliveryBatchResult()
await apply(
transaction,
tuple(
DeliveryProjection(_PAYLOAD.validate_python(delivery.accounting_payload))
for delivery in claimed
),
)
await transaction.execute_raw(_APPLIED, tuple(delivery.delivery_id for delivery in claimed))
return DeliveryBatchResult(applied=len(claimed))
except Exception as error: # noqa: BLE001 # roll back all destinations before recording a retry
if not claimed:
raise
return await self._record_failure(claimed, error, max_attempts)
async def _record_failure(
self, claimed: tuple[_Delivery, ...], error: Exception, max_attempts: int
) -> DeliveryBatchResult:
parks: Final = len(claimed) == 1 and _is_permanent_failure(error)
async with self._database.tx(timeout=_TIMEOUT) as transaction:
updates: Final = tuple(
[
await self._record_row_failure(
transaction,
delivery,
type(error).__name__,
"parked" if parks and delivery.attempts + 1 >= max_attempts else "pending",
)
for delivery in claimed
]
)
return DeliveryBatchResult(
failed=len(claimed), parked=tuple(chain.from_iterable(updates)), error=type(error).__name__
)
@staticmethod
async def _record_row_failure(
transaction: LedgerTransaction, delivery: _Delivery, error: str, state: Literal["pending", "parked"]
) -> tuple[str, ...]:
updated: Final = _DELIVERIES.validate_python(
await transaction.query_raw(_FAILED, delivery.delivery_id, delivery.attempts, error, state)
)
return tuple(row.delivery_id for row in updated) if state == "parked" else ()

88
litellm/repositories/spend_log_insert.py Normal file
View file

@ -0,0 +1,88 @@
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from typing import Final
from pydantic import JsonValue, TypeAdapter, ValidationError
from typing_extensions import LiteralString
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_COLUMNS: Final[tuple[LiteralString, ...]] = (
"request_id",
"call_type",
"api_key",
"spend",
"total_tokens",
"prompt_tokens",
"completion_tokens",
"startTime",
"endTime",
"request_duration_ms",
"completionStartTime",
"model",
"model_id",
"model_group",
"custom_llm_provider",
"api_base",
"user",
"metadata",
"cache_hit",
"cache_key",
"request_tags",
"team_id",
"organization_id",
"end_user",
"requester_ip_address",
"messages",
"response",
"session_id",
"status",
"mcp_namespaced_tool_name",
"billing_agent_id",
"agent_id",
"proxy_server_request",
"litellm_call_id",
"created_at",
"updated_at",
)
_JSON_COLUMNS: Final = frozenset({"metadata", "request_tags", "messages", "response", "proxy_server_request"})
class RejectedSpendLogRow(ValueError):
"""A row the raw insert cannot encode, so it is isolated like a row Postgres refuses."""
def _json_value(column: str, value: object) -> object:
if isinstance(value, datetime):
return value.isoformat()
if column not in _JSON_COLUMNS or not isinstance(value, str):
return value
try:
return _JSON.validate_json(value)
except ValidationError as error:
raise RejectedSpendLogRow(f"Spend log column {column} is not valid JSON") from error
def spend_log_insert_statement(rows: Sequence[Mapping[str, object]]) -> tuple[LiteralString, str]:
if not rows:
raise ValueError("Spend log insert requires at least one row")
fields: Final = frozenset(rows[0]) | {"created_at", "updated_at"}
if not fields.issubset(_COLUMNS) or any(frozenset(row) | {"created_at", "updated_at"} != fields for row in rows):
raise RejectedSpendLogRow("Spend log insert requires matching known columns")
columns: Final = ", ".join('"' + column + '"' for column in _COLUMNS if column in fields)
now: Final = datetime.now(timezone.utc).isoformat()
values: Final = tuple(
{"created_at": now, "updated_at": now, **{column: _json_value(column, value) for column, value in row.items()}}
for row in rows
)
sql: Final = (
'INSERT INTO "LiteLLM_SpendLogs" ('
+ columns
+ ") SELECT "
+ columns
+ ' FROM jsonb_populate_recordset(NULL::"LiteLLM_SpendLogs", $1::jsonb) '
"ON CONFLICT DO NOTHING RETURNING request_id"
)
return sql, safe_dumps(values, non_finite_as_null=True)

115
litellm/repositories/table_repositories.py
View file

@ -7,9 +7,16 @@ These are thin wrappers for tables that do not (yet) need domain-specific query
methods; richer repositories live in their own modules.
"""
from collections.abc import Mapping, Sequence
from datetime import timedelta
from itertools import groupby
from typing import TYPE_CHECKING, Any, Final, Generic, Protocol
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.repositories.accounting_delivery_repository import LedgerDatabase, LedgerTransaction
from litellm.repositories.prisma_protocols import RowT_co, TableActions
if TYPE_CHECKING:
@ -98,9 +105,117 @@ class OrganizationMembershipRepository(PrismaTableRepository["prisma_models.Lite
table_name = "litellm_organizationmembership"
_SPEND_LOG_FIELDS: Final = TypeAdapter(dict[str, object])
def _accounting_payload(value: object) -> Mapping[str, object]:
return (
_SPEND_LOG_FIELDS.validate_json(value) if isinstance(value, str) else _SPEND_LOG_FIELDS.validate_python(value)
)
_ACCOUNTING_ONLY_FIELDS: Final = frozenset({"accounting_payload", "accounting_log_disabled", "accounting_delivery_id"})
def _settles_once(row: Mapping[str, object]) -> bool:
"""A batch or video cost row: update_database gives every other ledger row its own delivery id."""
return "accounting_delivery_id" not in row
def _delivery_id(row: Mapping[str, object]) -> str:
return str(row["accounting_delivery_id"]) if not _settles_once(row) else str(row["request_id"])
class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs"]):
table_name = "litellm_spendlogs"
async def insert_with_deliveries(
self, rows: Sequence[Mapping[str, object]], *, write_spend_logs: bool = True
) -> int:
legacy: Final = tuple(row for row in rows if "accounting_payload" not in row)
legacy_count: Final = await self.table.create_many(data=legacy, skip_duplicates=True) if legacy else 0
ledger: Final = tuple(
sorted(
{_delivery_id(row): row for row in reversed(rows) if "accounting_payload" in row}.values(),
key=_delivery_id,
)
)
if not ledger:
return legacy_count
database: Final[LedgerDatabase] = self.prisma_client.writer_db
async with database.tx(timeout=timedelta(seconds=60)) as transaction:
accepted: Final = await self._insert_ledger(transaction, ledger, write_spend_logs=write_spend_logs)
return legacy_count + accepted
@staticmethod
async def _insert_ledger(
transaction: LedgerTransaction, rows: Sequence[Mapping[str, object]], *, write_spend_logs: bool
) -> int:
from litellm.proxy.db.db_span import db_span
from litellm.repositories.spend_log_insert import spend_log_insert_statement
payloads: Final = tuple(
{
"delivery_id": _delivery_id(row),
"request_id": row["request_id"],
"accounting_payload": _accounting_payload(row["accounting_payload"]),
"is_batch": _settles_once(row),
}
for row in rows
)
inserted_deliveries: Final = await transaction.query_raw(
'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) '
"SELECT delivery_id, request_id, accounting_payload, is_batch FROM jsonb_to_recordset($1::jsonb) "
"AS inputs(delivery_id text, request_id text, accounting_payload jsonb, is_batch boolean) "
"ON CONFLICT (delivery_id) DO NOTHING RETURNING delivery_id",
safe_dumps(payloads, non_finite_as_null=True),
)
accepted_ids: Final = frozenset(str(row["delivery_id"]) for row in inserted_deliveries)
charged_by_legacy: Final = await SpendLogsRepository._jobs_charged_by_legacy(
transaction,
tuple(_delivery_id(row) for row in rows if _settles_once(row) and _delivery_id(row) in accepted_ids),
)
if charged_by_legacy:
await transaction.execute_raw(
'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = ANY($1::text[])', list(charged_by_legacy)
)
accepted: Final = accepted_ids - charged_by_legacy
local_rows: Final = tuple(
{key: value for key, value in row.items() if key not in _ACCOUNTING_ONLY_FIELDS}
for row in rows
if _delivery_id(row) in accepted and row.get("accounting_log_disabled") is not True
)
if write_spend_logs and local_rows:
groups: Final = tuple(
tuple(group)
for _, group in groupby(
sorted(local_rows, key=lambda row: tuple(sorted(row))), key=lambda row: tuple(sorted(row))
)
)
async with db_span("insert_spend_logs", "LiteLLM_SpendLogs"):
for group in groups:
await transaction.query_raw(*spend_log_insert_statement(group))
return len(accepted)
@staticmethod
async def _jobs_charged_by_legacy(transaction: LedgerTransaction, cost_row_ids: Sequence[str]) -> frozenset[str]:
"""Batches and videos a pre-ledger poll already charged through their SpendLogs cost row."""
if not cost_row_ids:
return frozenset()
rows: Final = await transaction.query_raw(
'SELECT DISTINCT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1::text[]) '
"AND status = 'success' AND spend > 0",
list(cost_row_ids),
)
return frozenset(str(row["request_id"]) for row in rows)
async def has_delivery(self, delivery_id: str) -> bool:
database: Final[LedgerTransaction] = self.prisma_client.writer_db
rows: Final = await database.query_raw(
'SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE delivery_id = $1', delivery_id
)
return bool(rows)
class BudgetWindowSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_BudgetWindowSpend"]):
table_name = "litellm_budgetwindowspend"

42
litellm/types/proxy/model_deprecation.py
View file

@ -1,9 +1,10 @@
from __future__ import annotations
from datetime import date, datetime
from enum import Enum
from typing import Final, Literal
from pydantic import Field
from pydantic import ConfigDict, Field
from litellm.types.llms.base import LiteLLMBaseModel
@ -18,6 +19,10 @@ DeprecationStatus = Literal["upcoming", "imminent", "deprecated"]
class ModelDeprecationInfo(LiteLLMBaseModel):
model_name: str = Field(description="The public name of the model on the proxy (model_group).")
team_public_model_name: str | None = Field(
default=None,
description="The model name visible to the team, when configured.",
)
litellm_model: str | None = Field(
default=None,
description="The underlying litellm model string the deprecation date is sourced from.",
@ -59,3 +64,38 @@ class ModelDeprecationResponse(LiteLLMBaseModel):
)
warn_within_days: int = Field(description="The window (in days) used to bucket 'imminent' models.")
checked_at: datetime = Field(description="UTC timestamp when the deprecation snapshot was generated.")
class DeprecationAlertStatus(str, Enum):
OPEN = "open"
RESOLVED = "resolved"
DISMISSED = "dismissed"
class DeprecationAlertUpdate(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
model: str = Field(min_length=1, max_length=1024)
deprecation_date: date
status: DeprecationAlertStatus
last_seen_date: date
last_seen_requests: int = Field(ge=0)
class DeprecationAlertState(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
model: str
deprecation_date: date
status: DeprecationAlertStatus = DeprecationAlertStatus.OPEN
last_seen_date: date | None = None
last_seen_requests: int = 0
updated_at: datetime | None = None
updated_by: str | None = None
class DeprecationTagUsage(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
api_key: str
tag: str

1
litellm/types/utils.py
View file

@ -224,6 +224,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_adaptive_thinking: bool | None
supports_legacy_thinking: ReadOnly[bool | None]
thinking_always_on: ReadOnly[bool | None]
thinking_on_by_default: ReadOnly[bool | None]
supports_tool_search: bool | None
supports_mid_conversation_system: bool | None
supports_url_context: bool | None

1
litellm/utils.py
View file

@ -7599,6 +7599,7 @@ def get_model_info_helper(
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
supports_legacy_thinking=_model_info.get("supports_legacy_thinking", None),
thinking_always_on=_model_info.get("thinking_always_on", None),
thinking_on_by_default=_model_info.get("thinking_on_by_default", None),
supports_tool_search=_model_info.get("supports_tool_search", None),
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None),

255
model_prices_and_context_window.json
View file

@ -2125,6 +2125,7 @@
"anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -2213,6 +2214,7 @@
"global.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -2301,6 +2303,7 @@
"us.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2389,6 +2392,7 @@
"eu.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2472,6 +2476,7 @@
"au.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2560,6 +2565,7 @@
"jp.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
@ -2958,6 +2964,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3002,6 +3009,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3046,6 +3054,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3090,6 +3099,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3134,6 +3144,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3178,6 +3189,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -4063,6 +4075,7 @@
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
"litellm_provider": "azure_ai",
@ -4204,6 +4217,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -16507,6 +16521,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -17042,6 +17057,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -21910,6 +21926,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
@ -22068,6 +22085,7 @@
"prompt_cache_min_tokens": 1024,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_mid_conversation_system": true,
@ -36308,6 +36326,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3.2e-05,
"output_cost_per_image_token_batches": 1.6e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
@ -36331,6 +36350,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3.2e-05,
"output_cost_per_image_token_batches": 1.6e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
@ -36350,6 +36370,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/images/generations",
@ -36385,6 +36406,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
@ -36420,6 +36442,7 @@
"input_cost_per_image_token_batches": 4e-06,
"input_cost_per_token_batches": 2.5e-06,
"output_cost_per_image_token": 3e-05,
"output_cost_per_image_token_batches": 1.5e-05,
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
@ -39774,6 +39797,7 @@
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_image_token": 8e-06,
"output_cost_per_image_token_batches": 4e-06,
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/images/generations",
@ -50486,6 +50510,7 @@
"prompt_cache_min_tokens": 1024,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -50563,6 +50588,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -52968,6 +52994,7 @@
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_creation_input_token_cost_batches": 3.125e-06,
@ -53006,6 +53033,7 @@
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_creation_input_token_cost_batches": 3.125e-06,
@ -53255,6 +53283,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -59906,6 +59935,7 @@
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_image_token": 3.2e-05,
"output_cost_per_image_token_batches": 1.6e-05,
"output_cost_per_token": 1e-05,
"output_cost_per_token_batches": 5e-06,
"source": "https://developers.openai.com/api/docs/pricing",
@ -60629,6 +60659,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -64719,7 +64750,8 @@
{
"name": "claude-family-baseline",
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
"fill_missing_for_providers": ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai-anthropic_models"],
"description": "Any Claude family-major id with an optional minor and an optional 8-digit date suffix, under any provider namespace (bare, bedrock-dotted, vertex, databricks, ...), so bare majors like claude-newfamily-5 get the same baseline as claude-newfamily-5-1. Carries the model-family facts every Claude shares; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number. On mapped Claude entries from Anthropic, Azure Foundry, Bedrock and Vertex it only fills the keys an entry leaves out, so the entry's own values always win.",
"model_info": {
"mode": "chat",
"max_input_tokens": 200000,
@ -64762,6 +64794,14 @@
"thinking_always_on": true
}
},
{
"name": "claude-thinking-on-by-default",
"pattern": "claude-[a-z]+-[5-9](?!\\d)",
"description": "Claude at major 5 or higher, in any id shape that contains claude-<family>-<major>. Anthropic's thinking docs list every Claude 5+ model as thinking by default (on or always on) when the request sends no thinking param, and every 4.x as off. Two-digit majors are deliberately not matched so ids like claude-opus-41 (4.1) are not read as major 41. The response_format json tool is then left unforced, since a forced tool call skips the model's thinking.",
"model_info": {
"thinking_on_by_default": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|[5-9](?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",
@ -69285,6 +69325,7 @@
"output_cost_per_token": 1.2e-05,
"prompt_cache_min_tokens": 1024,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -69360,6 +69401,7 @@
"output_cost_per_token": 3e-05,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -69616,6 +69658,7 @@
"output_cost_per_token": 1.2e-05,
"prompt_cache_min_tokens": 1024,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -69681,6 +69724,7 @@
"output_cost_per_token": 3e-05,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -75432,6 +75476,7 @@
"apac.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
@ -75548,6 +75593,7 @@
"max_output_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77052,6 +77098,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77104,6 +77151,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77148,6 +77196,7 @@
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77195,6 +77244,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77231,6 +77281,7 @@
"max_output_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77287,6 +77338,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77333,6 +77385,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77373,6 +77426,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77414,6 +77468,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77460,6 +77515,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77511,6 +77567,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77557,6 +77614,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77617,6 +77675,7 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77668,6 +77727,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -77718,6 +77778,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -77762,6 +77823,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -79521,6 +79583,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -79735,6 +79798,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -79901,6 +79965,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -81886,6 +81951,7 @@
"prompt_cache_min_tokens": 512.0,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_adaptive_thinking": true,
"thinking_on_by_default": true,
"supports_anthropic_thinking_payload": true,
"supports_assistant_prefill": false,
"supports_function_calling": true,
@ -85912,5 +85978,192 @@
],
"supports_pdf_input": true,
"supports_vision": true
},
"bedrock/us-gov-east-1/moonshotai.kimi-k3": {
"cache_creation_input_token_cost": 4.95e-06,
"cache_creation_input_token_cost_flex": 2.475e-06,
"cache_creation_input_token_cost_priority": 8.6625e-06,
"cache_read_input_token_cost": 3.96e-07,
"cache_read_input_token_cost_flex": 1.98e-07,
"cache_read_input_token_cost_priority": 6.93e-07,
"input_cost_per_token": 3.96e-06,
"input_cost_per_token_batches": 1.98e-06,
"input_cost_per_token_flex": 1.98e-06,
"input_cost_per_token_priority": 6.93e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 868928,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.98e-05,
"output_cost_per_token_batches": 9.9e-06,
"output_cost_per_token_flex": 9.9e-06,
"output_cost_per_token_priority": 3.465e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_prompt_cache_breakpoint": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_regex_lookaround": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"token_count_framing": 64.0,
"token_count_tokenizer": "kimi_k3"
},
"bedrock/us-gov-west-1/moonshotai.kimi-k3": {
"cache_creation_input_token_cost": 4.95e-06,
"cache_creation_input_token_cost_flex": 2.475e-06,
"cache_creation_input_token_cost_priority": 8.6625e-06,
"cache_read_input_token_cost": 3.96e-07,
"cache_read_input_token_cost_flex": 1.98e-07,
"cache_read_input_token_cost_priority": 6.93e-07,
"input_cost_per_token": 3.96e-06,
"input_cost_per_token_batches": 1.98e-06,
"input_cost_per_token_flex": 1.98e-06,
"input_cost_per_token_priority": 6.93e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 868928,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.98e-05,
"output_cost_per_token_batches": 9.9e-06,
"output_cost_per_token_flex": 9.9e-06,
"output_cost_per_token_priority": 3.465e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json",
"supports_audio_input": false,
"supports_function_calling": true,
"supports_prompt_cache_breakpoint": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_regex_lookaround": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true,
"token_count_framing": 64.0,
"token_count_tokenizer": "kimi_k3"
},
"bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": {
"deprecation_date": "2026-09-10",
"deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 195904,
"max_output_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json",
"successor_model": "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/xai.grok-4.7": {
"cache_read_input_token_cost": 6.6e-07,
"cache_read_input_token_cost_flex": 3.3e-07,
"cache_read_input_token_cost_priority": 1.155e-06,
"input_cost_per_token": 2.64e-06,
"input_cost_per_token_batches": 1.32e-06,
"input_cost_per_token_flex": 1.32e-06,
"input_cost_per_token_priority": 4.62e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 7.92e-06,
"output_cost_per_token_batches": 3.96e-06,
"output_cost_per_token_flex": 3.96e-06,
"output_cost_per_token_priority": 1.386e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"use_openai_responses_path": true
},
"bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": {
"deprecation_date": "2026-09-10",
"deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-3-haiku.md",
"input_cost_per_token": 3e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 195904,
"max_output_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json",
"successor_model": "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0",
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/xai.grok-4.7": {
"cache_read_input_token_cost": 6.6e-07,
"cache_read_input_token_cost_flex": 3.3e-07,
"cache_read_input_token_cost_priority": 1.155e-06,
"input_cost_per_token": 2.64e-06,
"input_cost_per_token_batches": 1.32e-06,
"input_cost_per_token_flex": 1.32e-06,
"input_cost_per_token_priority": 4.62e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 7.92e-06,
"output_cost_per_token_batches": 3.96e-06,
"output_cost_per_token_flex": 3.96e-06,
"output_cost_per_token_priority": 1.386e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-west-1/index.json",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"use_openai_responses_path": true
},
"bedrock_mantle/xai.grok-4.7": {
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_flex": 2.75e-07,
"cache_read_input_token_cost_priority": 9.625e-07,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_batches": 1.1e-06,
"input_cost_per_token_flex": 1.1e-06,
"input_cost_per_token_priority": 3.85e-06,
"litellm_provider": "bedrock_mantle",
"max_input_tokens": 524288,
"max_output_tokens": 524288,
"mode": "chat",
"output_cost_per_token": 6.6e-06,
"output_cost_per_token_batches": 3.3e-06,
"output_cost_per_token_flex": 3.3e-06,
"output_cost_per_token_priority": 1.155e-05,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"use_openai_responses_path": true
}
}

7
pyproject.toml
View file

@ -155,7 +155,8 @@ stt-vertex-chirp = [
stt-nvidia-riva = [
# NVIDIA Riva STT provider (gRPC). These are imported lazily inside the
# provider handler so litellm core remains usable without them.
"nvidia-riva-client>=2.15.0",
"grpcio>=1.67.1,<2.0",
"protobuf>=5.26.1,<7.0",
"soundfile>=0.12.1",
"audioread>=3.0.1",
"numpy>=1.26.0",
@ -241,6 +242,7 @@ dev = [
"pytest-timeout==2.4.0",
"vcrpy==8.2.1",
"pytest-recording==0.13.4",
"nvidia-riva-client==2.16.0",
]
e2e-dev = [
"playwright==1.61.0",
@ -275,6 +277,9 @@ ci = [
# protobuf, Pillow is a compiled C extension).
"tenacity==8.5.0",
"google-generativeai==0.8.6",
# The Riva tests serve NVIDIA's own message definitions, which checks
# LiteLLM's hand-written copy of them on every run.
"nvidia-riva-client==2.16.0",
"Pillow==12.3.0; python_version == '3.10'",
"Pillow==13.0.0; python_version >= '3.11'",
# Azure batch E2E tests still import psycopg2 directly.

17
schema.prisma
View file

@ -2141,6 +2141,23 @@ model LiteLLM_AccountingDelivery {
@@index([target, state, next_attempt_at])
}
model LiteLLM_SpendLogDelivery {
delivery_id String @id @default(dbgenerated("gen_random_uuid()::text"))
request_id String
state AccountingDeliveryState @default(pending)
accounting_payload Json
is_batch Boolean
attempts Int @default(0)
next_attempt_at DateTime @default(now())
applied_at DateTime?
last_error String?
created_at DateTime @default(now())
@@index([state, next_attempt_at])
@@index([state, applied_at])
@@index([request_id])
}
model LiteLLM_SubscriptionAccountTable {
subscription_account_id String @id @default(uuid())
custom_llm_provider String

16
tests/integration/_support/riva.py
View file

@ -39,6 +39,7 @@ class RivaCall:
rpc: Rpc
audio: bytes
cancelled: bool
authorization: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
@ -68,6 +69,10 @@ def pcm_bytes(seconds: float) -> int:
return int(seconds * SAMPLE_RATE_HZ) * 2
def _authorization(context: grpc.ServicerContext) -> tuple[str, ...]:
return tuple(str(value) for key, value in context.invocation_metadata() if key == "authorization")
def carries_clip(audio: bytes, seconds: float) -> bool:
sent: Final = _clip_samples(seconds)
if len(audio) != len(sent) * 2:
@ -100,9 +105,9 @@ class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer):
released: Final = threading.Event()
context.add_callback(released.set)
released.wait(timeout=_STALL_LIMIT_SECONDS)
self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, cancelled=not context.is_active()))
self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, not context.is_active(), _authorization(context)))
return riva_asr_pb2.RecognizeResponse()
self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, cancelled=False))
self._calls.put(RivaCall(Rpc.RECOGNIZE, request.audio, False, _authorization(context)))
return riva_asr_pb2.RecognizeResponse(
results=[riva_asr_pb2.SpeechRecognitionResult(alternatives=[self._alternative(request.config)])]
)
@ -111,13 +116,12 @@ class _Asr(riva_asr_pb2_grpc.RivaSpeechRecognitionServicer):
self, request_iterator: Iterable[riva_asr_pb2.StreamingRecognizeRequest], context: grpc.ServicerContext
) -> Iterator[riva_asr_pb2.StreamingRecognizeResponse]:
if self._offline_only:
self._calls.put(RivaCall(Rpc.STREAMING, b"", cancelled=False))
self._calls.put(RivaCall(Rpc.STREAMING, b"", False, _authorization(context)))
context.abort(grpc.StatusCode.INVALID_ARGUMENT, OFFLINE_ONLY_REFUSAL)
requests: Final = tuple(request_iterator)
config: Final = requests[0].streaming_config.config if requests else riva_asr_pb2.RecognitionConfig()
self._calls.put(
RivaCall(Rpc.STREAMING, b"".join(request.audio_content for request in requests), cancelled=False)
)
audio: Final = b"".join(request.audio_content for request in requests)
self._calls.put(RivaCall(Rpc.STREAMING, audio, False, _authorization(context)))
yield riva_asr_pb2.StreamingRecognizeResponse(
results=[riva_asr_pb2.StreamingRecognitionResult(alternatives=[self._alternative(config)], is_final=True)]
)

512
tests/integration/database/test_spendlogs_delivery.py Normal file
View file

@ -0,0 +1,512 @@
import asyncio
import os
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from pathlib import Path
from typing import Final
from urllib.parse import urlsplit, urlunsplit
import psycopg
import pytest
import pytest_asyncio
from prisma import Prisma
from litellm.repositories.accounting_delivery_repository import (
AccountingDeliveryRepository,
DeliveryProjection,
LedgerTransaction,
)
from litellm.repositories.spend_log_insert import spend_log_insert_statement
from tests.integration._support.database import scratch_database
_MIGRATIONS: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/migrations"
@pytest.fixture
def delivery_database(request: pytest.FixtureRequest) -> Iterator[psycopg.Connection[tuple[object, ...]]]:
with scratch_database("spendlogs_delivery") as url, psycopg.connect(url, autocommit=True) as connection:
if getattr(request, "param", "public") == "ledger_tenant":
connection.execute("CREATE SCHEMA ledger_tenant")
connection.execute("SET search_path TO ledger_tenant")
connection.execute((_MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql").read_text())
connection.execute("""CREATE TABLE "LiteLLM_SpendLogs" (
request_id text PRIMARY KEY, call_type text NOT NULL, spend float8 NOT NULL DEFAULT 0,
status text,
metadata jsonb DEFAULT '{}', "startTime" timestamp(3) NOT NULL,
created_at timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at timestamp(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
)""")
connection.execute((_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text())
yield connection
def _insert(
connection: psycopg.Connection[tuple[object, ...]], rows: Sequence[Mapping[str, object]], *, ledger: bool = True
) -> None:
from psycopg.types.json import Jsonb
statement: Final = spend_log_insert_statement(rows)
with connection.transaction():
inserted: Final = {
row[0] for row in connection.execute(statement[0].replace("$1", "%s"), (statement[1],)).fetchall()
}
for row in rows:
if ledger and row["request_id"] in inserted:
connection.execute(
'INSERT INTO "LiteLLM_SpendLogDelivery" (delivery_id, request_id, accounting_payload, is_batch) '
"VALUES (%s, %s, %s, %s)",
(row["request_id"], row["request_id"], Jsonb(dict(row)), row.get("call_type") == "aretrieve_batch"),
)
def _with_payload(row: Mapping[str, object]) -> Mapping[str, object]:
return {**row, "accounting_payload": dict(row)}
def _row(request_id: str) -> Mapping[str, object]:
return {
"request_id": request_id,
"call_type": "acompletion",
"spend": 0.5,
"startTime": "2026-11-02T12:00:00+00:00",
"metadata": '{"accounting_savings":{"compression":0.2}}',
}
@pytest.mark.parametrize("call_type", ["acompletion", "aretrieve_batch"])
@pytest.mark.parametrize("mode", ["local", "url", "disabled"])
@pytest.mark.parametrize("partitioned", [False, True])
@pytest.mark.asyncio
async def test_delivery_first_acceptance_deduplicates_all_storage_modes(
delivery_database, delivery_client, mode, partitioned, call_type
):
from types import SimpleNamespace
from litellm.repositories.table_repositories import SpendLogsRepository
if partitioned:
delivery_database.execute('ALTER TABLE "LiteLLM_SpendLogs" RENAME TO spend_template')
delivery_database.execute(
'CREATE TABLE "LiteLLM_SpendLogs" (LIKE spend_template INCLUDING DEFAULTS INCLUDING CONSTRAINTS, PRIMARY KEY(request_id, "startTime")) PARTITION BY RANGE ("startTime")'
)
delivery_database.execute('CREATE TABLE "LiteLLM_SpendLogs_p20261102" PARTITION OF "LiteLLM_SpendLogs" DEFAULT')
delivery_database.execute("DROP TABLE spend_template")
repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client))
row = {**_with_payload({**_row("shared"), "call_type": call_type}), "accounting_log_disabled": mode == "disabled"}
results = await asyncio.gather(
repository.insert_with_deliveries((row,), write_spend_logs=mode != "url"),
repository.insert_with_deliveries(
({**row, "startTime": "2026-11-03T12:00:00"},), write_spend_logs=mode != "url"
),
)
assert sorted(results) == [0, 1]
assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(mode == "local"),)
assert delivery_database.execute(
"SELECT accounting_payload->>'spend', state::text FROM \"LiteLLM_SpendLogDelivery\""
).fetchall() == [("0.5", "pending")]
if partitioned:
delivery_database.execute('DROP TABLE "LiteLLM_SpendLogs_p20261102"')
else:
delivery_database.execute('DELETE FROM "LiteLLM_SpendLogs"')
async def apply(transaction, deliveries):
await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(deliveries[0].row["spend"]))
worker = AccountingDeliveryRepository(delivery_client)
assert (await worker.apply_pending(apply)).applied == 1
assert (await worker.apply_pending(apply)).applied == 0
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,)
@pytest.mark.parametrize("mode", ["local", "url", "disabled"])
@pytest.mark.asyncio
async def test_duplicate_rows_in_one_flush_keep_first_accounting_payload(delivery_database, delivery_client, mode):
from types import SimpleNamespace
from litellm.repositories.table_repositories import SpendLogsRepository
repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client))
first = {**_with_payload(_row("repeated")), "accounting_log_disabled": mode == "disabled"}
later = {
**_with_payload({**_row("repeated"), "spend": 9.0, "startTime": "2026-11-03T12:00:00+00:00"}),
"accounting_log_disabled": mode == "disabled",
}
assert await repository.insert_with_deliveries((first, later), write_spend_logs=mode != "url") == 1
assert delivery_database.execute(
'SELECT accounting_payload FROM "LiteLLM_SpendLogDelivery"'
).fetchone() == (first["accounting_payload"],)
assert delivery_database.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == (
[(first["spend"],)] if mode == "local" else []
)
@pytest.mark.parametrize(("legacy_spend", "accepted"), [(0.5, 0), (0.0, 1)])
@pytest.mark.asyncio
async def test_batch_charged_before_the_ledger_is_not_charged_again(
delivery_database, delivery_client, legacy_spend, accepted
):
from types import SimpleNamespace
from litellm.repositories.table_repositories import SpendLogsRepository
batch: Final = {**_row("batch_legacy"), "call_type": "aretrieve_batch", "status": "success"}
_insert(delivery_database, ({**batch, "spend": legacy_spend},), ledger=False)
repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client))
assert await repository.insert_with_deliveries((_with_payload({**batch, "spend": 1.0}),)) == accepted
assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (accepted,)
assert delivery_database.execute('SELECT spend FROM "LiteLLM_SpendLogs"').fetchall() == [(legacy_spend,)]
@pytest.mark.parametrize("table", ["LiteLLM_SpendLogs", "LiteLLM_SpendLogDelivery"])
@pytest.mark.asyncio
async def test_transaction_a_rolls_back_payload_and_log_on_failure(delivery_database, delivery_client, table):
from types import SimpleNamespace
from prisma.errors import PrismaError
from psycopg import sql
from litellm.repositories.table_repositories import SpendLogsRepository
delivery_database.execute(
sql.SQL("ALTER TABLE {} ADD CONSTRAINT reject_poison CHECK (request_id <> 'poison')").format(
sql.Identifier(table)
)
)
repository = SpendLogsRepository(SimpleNamespace(db=delivery_client, writer_db=delivery_client))
with pytest.raises(PrismaError):
await repository.insert_with_deliveries(tuple(_with_payload(_row(name)) for name in ("good", "poison")))
assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,)
assert delivery_database.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogs"').fetchone() == (0,)
@pytest.mark.asyncio
async def test_worker_rolls_back_destinations_and_parks_only_isolated_failure(
delivery_database: psycopg.Connection[tuple[object, ...]],
) -> None:
url: Final = urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}"))
client: Final = Prisma(datasource={"url": url})
delivery_database.execute("CREATE TABLE totals (spend float8 NOT NULL)")
delivery_database.execute("INSERT INTO totals VALUES (0)")
_insert(delivery_database, (_row("bad"), _row("good")))
async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None:
await transaction.execute_raw(
"UPDATE totals SET spend = spend + $1::float8", sum(float(str(row.row["spend"])) for row in rows)
)
if any(row.row["request_id"] == "bad" for row in rows):
raise ValueError("Invalid projected destination")
await client.connect()
try:
repository: Final = AccountingDeliveryRepository(client)
failed: Final = await repository.apply_pending(apply, max_attempts=2)
assert failed.failed == 2
assert failed.parked == ()
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,)
assert delivery_database.execute(
'SELECT DISTINCT state::text, attempts FROM "LiteLLM_SpendLogDelivery"'
).fetchall() == [("pending", 1)]
delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP')
isolated: Final = await repository.apply_pending(apply, max_attempts=2)
assert isolated.parked == ("bad",)
applied: Final = await repository.apply_pending(apply, max_attempts=2)
assert applied.applied == 1
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,)
assert delivery_database.execute(
'SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id'
).fetchall() == [("bad", "parked"), ("good", "applied")]
assert (await repository.apply_pending(apply)).applied == 0
finally:
await client.disconnect()
@pytest.mark.asyncio
async def test_a_locked_database_never_parks_a_valid_delivery(delivery_database, delivery_client):
_insert(delivery_database, (_row("valid"),))
async def apply(transaction, deliveries):
await transaction.execute_raw("SET LOCAL lock_timeout = '50ms'")
await transaction.execute_raw(
"UPDATE totals SET spend = spend + $1::float8", sum(float(row.row["spend"]) for row in deliveries)
)
repository = AccountingDeliveryRepository(delivery_client)
with psycopg.connect(
urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}"))
) as locker:
locker.execute("LOCK TABLE totals IN ACCESS EXCLUSIVE MODE")
for _ in range(4):
result = await repository.apply_pending(apply, max_attempts=2)
assert result.failed == 1 and result.parked == ()
delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP')
assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [
("pending", 4)
]
assert (await repository.apply_pending(apply, max_attempts=2)).applied == 1
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,)
@pytest.mark.asyncio
async def test_an_unreachable_database_reported_as_a_data_error_never_parks_a_delivery(
delivery_database, delivery_client
):
from prisma.errors import DataError
_insert(delivery_database, (_row("valid"),))
async def unreachable(transaction, deliveries):
raise DataError(data={"user_facing_error": {"message": "Can't reach database server at `db`:`5432`"}})
repository = AccountingDeliveryRepository(delivery_client)
for _ in range(3):
assert (await repository.apply_pending(unreachable, max_attempts=2)).parked == ()
delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at = CURRENT_TIMESTAMP')
assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchall() == [
("pending", 3)
]
@pytest_asyncio.fixture(loop_scope="function")
async def delivery_client(delivery_database: psycopg.Connection[tuple[object, ...]]) -> AsyncIterator[Prisma]:
url: Final = urlunsplit(urlsplit(os.environ["DATABASE_URL"])._replace(path=f"/{delivery_database.info.dbname}"))
client: Final = Prisma(datasource={"url": url})
delivery_database.execute("CREATE TABLE totals (spend float8 NOT NULL)")
delivery_database.execute("INSERT INTO totals VALUES (0)")
await client.connect()
try:
yield client
finally:
await client.disconnect()
@pytest.mark.asyncio
async def test_two_workers_claim_disjoint_rows_and_apply_once(
delivery_database: psycopg.Connection[tuple[object, ...]],
delivery_client: Prisma,
) -> None:
_insert(delivery_database, (_row("first"), _row("second")))
entered: Final[asyncio.Queue[str]] = asyncio.Queue()
release: Final = asyncio.Event()
async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None:
assert len(rows) == 1
await entered.put(str(rows[0].row["request_id"]))
await release.wait()
await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"])))
repository: Final = AccountingDeliveryRepository(delivery_client)
first: Final = asyncio.create_task(repository.apply_pending(apply, batch_size=1))
first_id: Final = await asyncio.wait_for(entered.get(), timeout=10)
second: Final = asyncio.create_task(repository.apply_pending(apply, batch_size=1))
try:
second_id: Final = await asyncio.wait_for(entered.get(), timeout=10)
assert first_id != second_id
finally:
release.set()
results: Final = await asyncio.gather(first, second)
assert sum(result.applied for result in results) == 2
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (1.0,)
assert delivery_database.execute(
"SELECT COUNT(*) FROM \"LiteLLM_SpendLogDelivery\" WHERE state = 'applied'"
).fetchone() == (2,)
@pytest.mark.asyncio
async def test_cancelled_worker_rolls_back_and_restart_applies_once(
delivery_database: psycopg.Connection[tuple[object, ...]],
delivery_client: Prisma,
) -> None:
_insert(delivery_database, (_row("restart"),))
wrote: Final = asyncio.Event()
hold: Final = asyncio.Event()
async def interrupted(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None:
await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"])))
wrote.set()
await hold.wait()
repository: Final = AccountingDeliveryRepository(delivery_client)
task: Final = asyncio.create_task(repository.apply_pending(interrupted))
await asyncio.wait_for(wrote.wait(), timeout=10)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,)
assert delivery_database.execute('SELECT state::text, attempts FROM "LiteLLM_SpendLogDelivery"').fetchone() == (
"pending",
0,
)
async def complete(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None:
await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"])))
assert (await repository.apply_pending(complete)).applied == 1
assert (await repository.apply_pending(complete)).applied == 0
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,)
@pytest.mark.asyncio
async def test_worker_applies_accounting_after_spend_logs_are_deleted(
delivery_database: psycopg.Connection[tuple[object, ...]], delivery_client: Prisma
) -> None:
_insert(delivery_database, (_row("removed"),))
delivery_database.execute('DELETE FROM "LiteLLM_SpendLogs"')
async def apply(transaction: LedgerTransaction, rows: tuple[DeliveryProjection, ...]) -> None:
await transaction.execute_raw("UPDATE totals SET spend = spend + $1::float8", float(str(rows[0].row["spend"])))
repository: Final = AccountingDeliveryRepository(delivery_client)
assert (await repository.apply_pending(apply)).applied == 1
assert (await repository.apply_pending(apply)).applied == 0
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,)
@pytest.mark.asyncio
async def test_receipt_cleanup_keeps_pending_and_parked_rows_and_reports_lag(
delivery_database: psycopg.Connection[tuple[object, ...]], delivery_client: Prisma
) -> None:
_insert(delivery_database, tuple(_row(state) for state in ("pending", "parked", "applied")))
delivery_database.execute(
'UPDATE "LiteLLM_SpendLogDelivery" '
'SET state = request_id::"AccountingDeliveryState", '
"created_at = CURRENT_TIMESTAMP - INTERVAL '40 days', "
"applied_at = CURRENT_TIMESTAMP - INTERVAL '40 days'"
)
repository = AccountingDeliveryRepository(delivery_client)
assert await repository.cleanup_applied() == 1
metrics = await repository.metrics()
assert metrics.pending_count == 1
assert metrics.parked_count == 1
assert metrics.oldest_pending_seconds >= 40 * 86400
assert delivery_database.execute(
'SELECT request_id FROM "LiteLLM_SpendLogDelivery" ORDER BY request_id'
).fetchall() == [("parked",), ("pending",)]
@pytest.mark.parametrize("delivery_database", ["public", "ledger_tenant"], indirect=True)
@pytest.mark.parametrize("state", [None, "pending", "parked", "applied"])
def test_delivery_migration_replay_preserves_ledger_receipts(delivery_database, state):
if state is not None:
_insert(delivery_database, (_row("replay"),))
delivery_database.execute(
'UPDATE "LiteLLM_SpendLogDelivery" SET state=%s, attempts=3, last_error=%s',
(state, "retained failure"),
)
before = delivery_database.execute('SELECT * FROM "LiteLLM_SpendLogDelivery"').fetchall()
migration = (_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text()
delivery_database.execute(migration)
delivery_database.execute(migration)
assert delivery_database.execute('SELECT * FROM "LiteLLM_SpendLogDelivery"').fetchall() == before
assert delivery_database.execute('SELECT count(*) FROM "LiteLLM_SpendLogs"').fetchone() == (int(state is not None),)
@pytest.mark.parametrize("state", ["pending", "parked"])
def test_delivery_migration_preserves_unused_old_receipts(state):
with scratch_database("spendlogs_old_delivery") as url, psycopg.connect(url, autocommit=True) as connection:
connection.execute((_MIGRATIONS / "20261027000000_add_accounting_ledger_tables/migration.sql").read_text())
connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id text PRIMARY KEY)')
connection.execute(
'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state) VALUES (%s, %s, 1, %s)',
("old-event", "LiteLLM_UserTable", state),
)
connection.execute((_MIGRATIONS / "20261103010000_spendlogs_accounting_delivery/migration.sql").read_text())
assert connection.execute('SELECT event_id, state::text FROM "LiteLLM_AccountingDelivery"').fetchall() == [
("old-event", state)
]
assert connection.execute('SELECT COUNT(*) FROM "LiteLLM_SpendLogDelivery"').fetchone() == (0,)
@pytest.mark.asyncio
async def test_malformed_payload_is_parked_without_losing_other_deliveries(delivery_database, delivery_client):
_insert(delivery_database, (_row("bad"), _row("good")))
delivery_database.execute(
'UPDATE "LiteLLM_SpendLogDelivery" SET accounting_payload=$$ [] $$::jsonb WHERE request_id=$$bad$$'
)
async def apply(transaction, deliveries):
await transaction.execute_raw(
"UPDATE totals SET spend = spend + $1::float8", sum(float(row.row["spend"]) for row in deliveries)
)
repository = AccountingDeliveryRepository(delivery_client)
assert (await repository.apply_pending(apply, max_attempts=2)).failed == 2
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.0,)
delivery_database.execute('UPDATE "LiteLLM_SpendLogDelivery" SET next_attempt_at=CURRENT_TIMESTAMP')
assert (await repository.apply_pending(apply, max_attempts=2)).parked == ("bad",)
assert (await repository.apply_pending(apply, max_attempts=2)).applied == 1
assert delivery_database.execute("SELECT spend FROM totals").fetchone() == (0.5,)
@pytest.mark.asyncio
async def test_retention_keeps_batch_receipts_indefinitely(delivery_database, delivery_client):
_insert(delivery_database, (_row("normal"), {**_row("batch"), "call_type": "aretrieve_batch"}))
delivery_database.execute(
"UPDATE \"LiteLLM_SpendLogDelivery\" SET state='applied', applied_at=CURRENT_TIMESTAMP - INTERVAL '400 days'"
)
repository = AccountingDeliveryRepository(delivery_client)
assert await repository.cleanup_applied() == 1
assert await repository.cleanup_applied() == 0
assert delivery_database.execute('SELECT request_id FROM "LiteLLM_SpendLogDelivery"').fetchall() == [("batch",)]
metrics = await repository.metrics()
assert metrics.batch_count == 1
assert metrics.table_bytes > 0
@pytest.mark.parametrize("disabled", [False, True])
@pytest.mark.asyncio
async def test_ledger_batch_retrieves_charge_once_in_every_logging_mode(
delivery_database, delivery_client, monkeypatch, disabled
):
from types import SimpleNamespace
from litellm.proxy import proxy_server
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.utils import jsonify_object
completed = {
**_row("batch_completed"), "call_type": "aretrieve_batch", "status": "success", "spend": 0.5,
"metadata": '{"status":"success","private":"request detail"}',
"accounting_payload": {
"request_id": "batch_completed", "call_type": "aretrieve_batch", "status": "success", "spend": 0.5,
"metadata": {"accounting_savings": {"compression": 0.2}},
},
}
monkeypatch.setattr(proxy_server, "general_settings", {"spend_logs_metadata_fields": {"include": ["status"]}})
client = SimpleNamespace(db=delivery_client, writer_db=delivery_client, jsonify_object=jsonify_object)
writer = DBSpendUpdateWriter()
first = await writer._claim_settle_once_cost_row(payload=completed, prisma_client=client, disable_spend_logs=disabled)
again = await writer._claim_settle_once_cost_row(
payload={**completed, "startTime": "2026-11-05T12:00:00+00:00"}, prisma_client=client, disable_spend_logs=disabled
)
assert (first, again) == (True, False)
assert delivery_database.execute('SELECT spend, metadata FROM "LiteLLM_SpendLogs"').fetchall() == (
[] if disabled else [(0.5, {"status": "success"})]
)
assert delivery_database.execute(
'SELECT delivery_id, accounting_payload FROM "LiteLLM_SpendLogDelivery"'
).fetchall() == [("batch_completed", completed["accounting_payload"])]
@pytest.mark.parametrize("setting", ["retention_days", "batch_size"])
@pytest.mark.asyncio
async def test_invalid_cleanup_settings_preserve_receipts(delivery_database, delivery_client, setting):
_insert(delivery_database, (_row("protected"),))
with pytest.raises(ValueError, match="positive"):
await AccountingDeliveryRepository(delivery_client).cleanup_applied(**{setting: 0})
assert delivery_database.execute('SELECT request_id, state::text FROM "LiteLLM_SpendLogDelivery"').fetchall() == [
("protected", "pending")
]
@pytest.mark.parametrize("setting", ["max_attempts", "batch_size"])
@pytest.mark.asyncio
async def test_invalid_worker_settings_preserve_pending_work(delivery_database, delivery_client, setting):
from unittest.mock import AsyncMock
_insert(delivery_database, (_row("protected"),))
apply = AsyncMock()
with pytest.raises(ValueError, match="positive"):
await AccountingDeliveryRepository(delivery_client).apply_pending(apply, **{setting: 0})
apply.assert_not_awaited()
assert delivery_database.execute(
'SELECT request_id, state::text, attempts FROM "LiteLLM_SpendLogDelivery"'
).fetchall() == [("protected", "pending", 0)]

1437
tests/integration/database/test_spendlogs_ledger_parity.py Normal file
View file

File diff suppressed because it is too large Load diff

44
tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_overloaded_wire.py
View file

@ -140,6 +140,16 @@ def _assert_internal_server_error(response: httpx.Response) -> None:
assert "InternalServerError" in string_value(error["message"]), response.text
def _assert_openai_overloaded(response: httpx.Response) -> None:
assert response.status_code == 503, response.text
error: Final = _openai_error(response)
assert set(error) == {"message", "type", "param", "code"}, response.text
assert string_value(error["message"]), response.text
assert error["type"] == "service_unavailable_error", response.text
assert error["param"] is None, response.text
assert error["code"] == "server_is_overloaded", response.text
def _one_post(wire: Wire, target: str) -> None:
assert [(request.method, unquote(request.target)) for request in wire.drain()] == [("POST", target)]
@ -361,7 +371,7 @@ def test_messages_overloaded_frame_before_content_without_a_retry_budget_is_rela
]
def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: Gateway) -> None:
def test_chat_completions_overloaded_frame_before_content_answers_503(gateway: Gateway) -> None:
prompt: Final = _prompt()
with wire_server(_always(_overloaded_before_content(prompt))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
@ -370,24 +380,24 @@ def test_chat_completions_overloaded_frame_before_content_answers_500(gateway: G
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": prompt}], "stream": True, "num_retries": 0},
)
_assert_internal_server_error(response)
_assert_openai_overloaded(response)
_one_post(wire, "/v1/messages")
def test_chat_completions_529_answers_500_to_the_sync_sdk(gateway: Gateway) -> None:
def test_chat_completions_529_answers_503_to_the_sync_sdk(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
with pytest.raises(openai.APIStatusError) as caught:
_openai_sdk(gateway).chat.completions.create(
model=model, messages=[{"role": "user", "content": _prompt()}], extra_body={"num_retries": 0}
)
assert caught.value.status_code == 500, caught.value.response.text
_assert_internal_server_error(caught.value.response)
assert caught.value.status_code == 503, caught.value.response.text
_assert_openai_overloaded(caught.value.response)
_one_post(wire, "/v1/messages")
_assert_logged_failure(model, rows=1, error_class="InternalServerError", code="500")
_assert_logged_failure(model, rows=1, error_class="InternalServerError", code="503")
async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway: Gateway) -> None:
async def test_chat_completions_stream_529_answers_503_to_the_async_sdk(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
with pytest.raises(openai.APIStatusError) as caught:
@ -397,19 +407,31 @@ async def test_chat_completions_stream_529_answers_500_to_the_async_sdk(gateway:
stream=True,
extra_body={"num_retries": 0},
)
assert caught.value.status_code == 500, caught.value.response.text
_assert_internal_server_error(caught.value.response)
assert caught.value.status_code == 503, caught.value.response.text
_assert_openai_overloaded(caught.value.response)
_one_post(wire, "/v1/messages")
def test_completions_529_answers_503(gateway: Gateway) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/completions",
{"model": model, "prompt": _prompt(), "max_tokens": 16, "num_retries": 0},
)
_assert_openai_overloaded(response)
_one_post(wire, "/v1/messages")
@pytest.mark.parametrize("shape", ("non-stream", "stream"))
def test_responses_529_answers_500(gateway: Gateway, shape: Shape) -> None:
def test_responses_529_answers_503(gateway: Gateway, shape: Shape) -> None:
with wire_server(_always(status_reply(529))) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST", "/v1/responses", {"model": model, "input": _prompt(), "stream": shape == "stream", "num_retries": 0}
)
_assert_internal_server_error(response)
_assert_openai_overloaded(response)
_one_post(wire, "/v1/messages")

8
tests/integration/observability/test_anthropic_overloaded_failure_signals.py
View file

@ -215,7 +215,7 @@ def _until(rig: _Rig, name: str, model: str) -> tuple[Sample, ...]:
(
pytest.param("/v1/messages", _MESSAGES_ANTHROPIC, 529, "500", "InternalServerError", id="messages-anthropic"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, "503", "ServiceUnavailableError", id="messages-bedrock"),
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 500, "500", "InternalServerError", id="chat-anthropic"),
pytest.param("/v1/chat/completions", _CHAT_ANTHROPIC, 503, "500", "InternalServerError", id="chat-anthropic"),
),
)
def test_the_failed_request_metric_counts_the_answered_status_and_the_deployment_metric_the_deployment_status(
@ -245,7 +245,7 @@ def _server_spans(rig: _Rig, since: int, trace_id: str) -> tuple[Span, ...]:
("route", "model", "answered"),
(
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat"),
),
)
def test_the_server_span_carries_the_status_the_route_answered(
@ -313,7 +313,7 @@ def _datadog_status_codes(rig: _Rig, alias: str, expected: int) -> tuple[tuple[s
(
pytest.param("/v1/messages", _DATADOG_MESSAGES_ANTHROPIC, 529, "500", id="messages-anthropic"),
pytest.param("/v1/messages", _DATADOG_MESSAGES_BEDROCK, 503, "503", id="messages-bedrock"),
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 500, "500", id="chat-anthropic"),
pytest.param("/v1/chat/completions", _DATADOG_CHAT_ANTHROPIC, 503, "500", id="chat-anthropic"),
),
)
def test_the_datadog_proxy_failure_log_carries_the_answered_status_and_the_llm_failure_log_the_deployment_status(
@ -361,7 +361,7 @@ def test_a_fail_closed_guardrail_outage_is_answered_and_recorded_with_the_routes
(
pytest.param("/v1/messages", _TRACED_MESSAGES, 529, id="messages-anthropic"),
pytest.param("/v1/messages", _MESSAGES_BEDROCK, 503, id="messages-bedrock"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 500, id="chat-anthropic"),
pytest.param("/v1/chat/completions", _TRACED_CHAT, 503, id="chat-anthropic"),
),
)
def test_the_otel_v2_server_span_carries_the_status_the_route_answered(

20
tests/integration/providers/test_anthropic_tool_search_result_matrix_wire.py
View file

@ -482,15 +482,20 @@ def test_duplicated_search_result_blocks_rebuild_the_call_once(gateway: Gateway,
@pytest.mark.parametrize(
("status", "kind", "caller_status"),
("status", "kind", "caller_status", "caller_error_type", "caller_error_code"),
(
pytest.param(400, "invalid_request_error", 400, id="400"),
pytest.param(401, "authentication_error", 401, id="401"),
pytest.param(529, "overloaded_error", 500, id="529"),
pytest.param(400, "invalid_request_error", 400, None, None, id="400"),
pytest.param(401, "authentication_error", 401, None, None, id="401"),
pytest.param(529, "overloaded_error", 503, "service_unavailable_error", "server_is_overloaded", id="529"),
),
)
def test_provider_error_on_the_replay_reaches_the_caller(
gateway: Gateway, status: int, kind: str, caller_status: int
gateway: Gateway,
status: int,
kind: str,
caller_status: int,
caller_error_type: str | None,
caller_error_code: str | None,
) -> None:
marker: Final = uuid.uuid4().hex
@ -508,6 +513,11 @@ def test_provider_error_on_the_replay_reaches_the_caller(
received: Final = wire.drain()
assert second.status_code == caller_status, second.text
assert f"scripted {kind} {marker}" in second.text, second.text
if caller_error_type is not None:
payload: Final = JSON_OBJECT.validate_json(second.content)
error: Final = JSON_OBJECT.validate_python(payload["error"])
assert error["type"] == caller_error_type, second.text
assert error["code"] == caller_error_code, second.text
assert len(received) >= 2, received
assert wire_assistant(received[1]) == list(SEARCH_TURN), received[1].body
assert gateway.client.get("/health/liveliness").status_code == 200

12
tests/integration/providers/test_bedrock_native_structured_output_wire.py
View file

@ -622,10 +622,15 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output
@pytest.mark.parametrize("route", ["converse", "invoke"])
def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gateway: Gateway, route: str) -> None:
@pytest.mark.parametrize(
("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"]
)
def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default(
gateway: Gateway, route: str, backend: str, thinks_by_default: bool
) -> None:
marker, marker_hex = _marker()
with wire_server(_bedrock) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire, route, _HAIKU)
model: Final = _deployment(scenario, wire, route, backend)
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_response_format(_SCHEMA))
)
@ -634,6 +639,9 @@ def test_bedrock_native_model_without_thinking_keeps_the_forced_json_tool(gatewa
assert not message.get("tool_calls"), response.text
_, body = _one_call(wire, marker_hex)
invoke: Final = route == "invoke"
if thinks_by_default:
_assert_native_without_tools(body, invoke, _SCHEMA)
return
assert _native_schema(body, invoke) is None, body
assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body

9
tests/integration/providers/test_nvidia_riva_wire.py
View file

@ -135,3 +135,12 @@ def test_a_stalled_offline_recognize_answers_408_at_the_deployment_timeout(gatew
assert timed_out.status_code == 408, timed_out.text
calls: Final = eventually(server.drain, lambda observed: len(observed) > 0, seconds=30)
assert all(call.rpc is Rpc.RECOGNIZE and call.cancelled for call in calls), calls
@pytest.mark.parametrize("riva_offline", (False, True), ids=("streaming", "offline"))
def test_the_deployment_api_key_reaches_riva_as_exactly_one_bearer_token(gateway: Gateway, riva_offline: bool) -> None:
with riva_server(_WORDS) as server, gateway.scenario() as scenario:
model: Final = _riva_model(scenario, server, api_key="riva-deployment-key", riva_offline=riva_offline)
served: Final = _transcribe(gateway, scenario.key(), model)
assert served.status_code == 200, served.text
assert [call.authorization for call in server.drain()] == [("Bearer riva-deployment-key",)]

58
tests/integration/providers/test_vertex_anthropic_structured_output_wire.py
View file

@ -34,14 +34,20 @@ from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.shares_worker_state
_JSON_TOOL: Final = "json_tool_call"
_UNFORCED_JSON_TOOL_DESCRIPTION: Final = (
"When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text."
)
_USER_TOOL: Final = "lookup"
_STRUCTURED_OUTPUTS_BETA: Final = "structured-outputs-2025-11-13"
_TOOL_MODEL: Final = "claude-sonnet-5"
_EFFORT_GATED_MODEL: Final = "claude-haiku-4-5"
_NATIVE_MODEL: Final = "claude-opus-5-5"
_FORCED_TOOL_NATIVE_MODEL: Final = "claude-haiku-5-5"
_FORCED_TOOL_NATIVE_MODEL: Final = "claude-opus-4-8"
_THINKING_BY_DEFAULT_NATIVE_MODEL: Final = "claude-haiku-5-5"
_THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8"
_UNMAPPED_MODEL: Final = "claude-scripted-9"
_BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5"
_BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8"
_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8"
_THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl"
_PROJECT: Final = "scripted-project"
@ -608,11 +614,15 @@ def _one_delivery(wire: Wire, marker_hex: str) -> _Delivery:
return sent
def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]:
def _json_tool(delivery: _Delivery) -> dict[str, JsonValue]:
tools: Final = tuple(object_value(tool) for tool in list_value(delivery.body["tools"]))
(json_tool,) = tuple(tool for tool in tools if tool["name"] == _JSON_TOOL)
assert json_tool.keys() == {"name", "input_schema"}, json_tool
return object_value(json_tool["input_schema"])
assert json_tool.keys() - {"description"} == {"name", "input_schema"}, json_tool
return json_tool
def _json_tool_schema(delivery: _Delivery) -> dict[str, JsonValue]:
return object_value(_json_tool(delivery)["input_schema"])
def _folded_format(delivery: _Delivery) -> dict[str, JsonValue]:
@ -830,10 +840,10 @@ def _family_deployment(gateway: Gateway, scenario: Scenario, api_base: str, memb
case "azure_ai_unmapped":
return scenario.model(model=f"azure_ai/{_UNMAPPED_MODEL}", api_base=api_base, api_key=_AZURE_KEY)
case "bedrock_invoke":
return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_INVOKE_MODEL)
return _bedrock_invoke_deployment(scenario, api_base, _BEDROCK_TOOL_PATH_MODEL)
case "bedrock_mantle":
return scenario.model(
model=f"bedrock_mantle/{_BEDROCK_INVOKE_MODEL}",
model=f"bedrock_mantle/{_BEDROCK_TOOL_PATH_MODEL}",
api_base=api_base,
api_key=_MANTLE_KEY,
aws_region_name="us-east-1",
@ -1448,21 +1458,24 @@ def test_tool_path_stream_cut_before_the_tool_block_closes_still_delivers_the_an
_one_delivery(wire, marker_hex)
@pytest.mark.parametrize("accepts_forced_tools", [True, False], ids=["accepts_forced_tools", "rejects_forced_tools"])
def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_accepts_forced_tools(
gateway: Gateway, accepts_forced_tools: bool
@pytest.mark.parametrize(
("tool_model", "forced"),
[(_THINKING_OFF_TOOL_MODEL, True), (_TOOL_MODEL, False), (None, False)],
ids=["accepts_forced_tools", "thinks_by_default", "rejects_forced_tools"],
)
def test_vertex_chat_json_schema_forces_the_json_tool_only_when_the_model_will_not_think(
gateway: Gateway, tool_model: str | None, forced: bool
) -> None:
marker, marker_hex = _marker("plain")
backend: Final = (
_TOOL_MODEL if accepts_forced_tools else _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude")
)
backend: Final = tool_model or _claude_rejecting_forced_tools(gateway, "vertex_ai/", "claude")
with wire_server(_peer) as wire, gateway.scenario() as scenario:
model: Final = _vertex_deployment(gateway, scenario, wire.url, backend)
response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker))
assert _chat_content(response) == _payload(marker_hex), response.text
sent: Final = _one_delivery(wire, marker_hex)
assert _json_tool_schema(sent)["properties"] == _SCHEMA["properties"], sent.body
assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if accepts_forced_tools else None), sent.body
assert sent.body.get("tool_choice") == (_FORCED_JSON_TOOL if forced else None), sent.body
assert _json_tool(sent).get("description") == (None if forced else _UNFORCED_JSON_TOOL_DESCRIPTION), sent.body
assert "output_config" not in sent.body and "output_format" not in sent.body, sent.body
@ -1566,6 +1579,25 @@ def test_native_model_json_schema_with_an_open_object_takes_the_json_tool_and_ke
sent: Final = _one_delivery(wire, marker_hex)
assert not _structured(sent.body), sent.body
assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body
assert sent.body.get("tool_choice") == _FORCED_JSON_TOOL, sent.body
@pytest.mark.parametrize("provider", ["anthropic", "azure_ai"])
def test_open_object_schema_takes_the_unforced_json_tool_on_a_model_that_thinks_by_default(
gateway: Gateway, provider: str
) -> None:
marker, marker_hex = _marker("values")
with wire_server(_peer) as wire, gateway.scenario() as scenario:
model: Final = _native_deployment(gateway, scenario, wire.url, provider, _THINKING_BY_DEFAULT_NATIVE_MODEL)
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, response_format=_VALUES_RESPONSE_FORMAT)
)
assert _chat_content(response) == _tool_input("values", marker_hex), response.text
sent: Final = _one_delivery(wire, marker_hex)
assert not _structured(sent.body), sent.body
assert _json_tool_schema(sent)["properties"] == _VALUES_SCHEMA["properties"], sent.body
assert _json_tool(sent).get("description") == _UNFORCED_JSON_TOOL_DESCRIPTION, sent.body
assert "tool_choice" not in sent.body, sent.body
@pytest.mark.parametrize(

10
tests/integration/routing/test_prompt_cache_pin_ttl.py
View file

@ -9,7 +9,7 @@ from typing import Final
import anthropic
import openai
import pytest
from integration._support.client import eventually, gateway_from_environment, held
from integration._support.client import eventually, gateway_from_environment, held, object_value
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.redis_process import owned_redis
from integration._support.wire import Wire, wire_server
@ -56,11 +56,12 @@ from integration.routing._prompt_cache_pin_support import (
vertex_peer,
wire_carries_one_hour,
)
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter
from redis import Redis
pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 300), pytest.mark.shares_worker_state]
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_CLAUDE: Final = "pin-claude"
_SLOW: Final = "pin-slow"
_MIXED: Final = "pin-mixed"
@ -539,7 +540,10 @@ def test_failed_first_turn_reports_the_error_and_writes_no_pin(rig: _Rig) -> Non
"/v1/chat/completions",
chat_body(_CLAUDE, _first_turn(new_marker(), f"{new_marker()} {OVERLOAD_TOKEN}", ONE_HOUR_MARK)),
)
assert response.status_code == 500, response.text
assert response.status_code == 503, response.text
error: Final = object_value(_JSON_OBJECT.validate_json(response.text)["error"])
assert error["type"] == "service_unavailable_error", response.text
assert error["code"] == "server_is_overloaded", response.text
assert "overloaded_error" in response.text, response.text
healthy, _ = post_chat(rig.owned.gateway, chat_body(_CLAUDE, _first_turn(new_marker(), new_marker(), None)))
spend_row_lands(healthy)

15
tests/integration/routing/test_reasoning_effort_levels_wire.py
View file

@ -1977,6 +1977,9 @@ _INVOKE_NATIVE_OUTPUT: Final[dict[str, JsonValue]] = {
"format": {"type": "json_schema", "schema": _NATIVE_OUTPUT_SCHEMA},
}
_JSON_TOOL: Final = "json_tool_call"
_UNFORCED_JSON_TOOL_DESCRIPTION: Final = (
"When you are ready to answer, call this tool with your final answer as its input. Do not reply in plain text."
)
_CONVERSE_TOOL_SCHEMA: Final[dict[str, JsonValue]] = {
key: value for key, value in _NATIVE_OUTPUT_SCHEMA.items() if key != "additionalProperties"
}
@ -1994,7 +1997,7 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields(
{
"toolSpec": {
"name": _JSON_TOOL,
"description": _JSON_TOOL,
"description": _UNFORCED_JSON_TOOL_DESCRIPTION,
"inputSchema": {"json": _CONVERSE_TOOL_SCHEMA},
}
}
@ -2004,7 +2007,15 @@ _CONVERSE_WITH_JSON_TOOL: Final = _with_fields(
)
_INVOKE_WITH_JSON_TOOL: Final = _with_fields(
_without(_invoke_chat("high", _ADAPTIVE, ()), "anthropic_beta"),
{"tools": [{"name": _JSON_TOOL, "input_schema": _NATIVE_OUTPUT_SCHEMA}]},
{
"tools": [
{
"name": _JSON_TOOL,
"description": _UNFORCED_JSON_TOOL_DESCRIPTION,
"input_schema": _NATIVE_OUTPUT_SCHEMA,
}
]
},
)

2
tests/integration/sdk/test_anthropic_json_tool_cut_stream_sync_sdk.py
View file

@ -27,7 +27,7 @@ _RESPONSE_FORMAT: Final[dict[str, JsonValue]] = {
"json_schema": {"name": "risk", "schema": _SCHEMA},
}
_ANTHROPIC_MODEL: Final = "anthropic/claude-scripted-9"
_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-sonnet-5"
_BEDROCK_INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-opus-4-8"
def _payload(marker_hex: str) -> dict[str, JsonValue]:

22
tests/integration/translation/chat_completions/response_format/test_chat_completions_response_format_anthropic.py
View file

@ -6,7 +6,7 @@ import pytest
from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import TranslationTestCase
from integration.translation.chat_completions.bases.anthropic import CLAUDE_HAIKU_5_5_TEST_CASE
from integration.translation.chat_completions.bases.anthropic import CLAUDE_OPUS_4_8_TEST_CASE
from integration.translation.runner import assert_translation
PLACE_SCHEMA: Final = {
@ -24,25 +24,25 @@ PLACE_TOOL_USE: Final = {
"caller": {"type": "direct"},
}
CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace(
CLAUDE_HAIKU_5_5_TEST_CASE,
CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE: Final = replace(
CLAUDE_OPUS_4_8_TEST_CASE,
scenario="json_schema",
litellm_request={
"model": "anthropic/claude-haiku-5-5",
"model": "anthropic/claude-opus-4-8",
"max_tokens": 128,
"messages": [{"role": "user", "content": PLACE_PROMPT}],
"response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}},
"cache": {"no-cache": True},
},
expected_provider_request={
"model": "claude-haiku-5-5",
"model": "claude-opus-4-8",
"messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}],
"max_tokens": 128,
"tool_choice": {"name": "json_tool_call", "type": "tool"},
"tools": [{"name": "json_tool_call", "input_schema": PLACE_SCHEMA}],
},
mock_provider_response={
"model": "claude-haiku-5-5",
"model": "claude-opus-4-8",
"id": "msg_011CgCoP2UR3vcM5ZWmEBd8D",
"type": "message",
"role": "assistant",
@ -68,7 +68,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace(
"id": ANY,
"created": ANY,
"object": "chat.completion",
"model": "anthropic/claude-haiku-5-5",
"model": "anthropic/claude-opus-4-8",
"choices": [
{
"finish_reason": "stop",
@ -97,11 +97,11 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE: Final = replace(
},
)
CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace(
CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE,
CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace(
CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE,
scenario="json_schema_tool_named_envelope",
mock_provider_response={
**CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE.mock_provider_response,
**CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE.mock_provider_response,
"content": [
{
**PLACE_TOOL_USE,
@ -114,7 +114,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE: Final = replace(
@pytest.mark.parametrize(
"case",
[CLAUDE_HAIKU_5_5_JSON_SCHEMA_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE],
[CLAUDE_OPUS_4_8_JSON_SCHEMA_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_TOOL_NAMED_ENVELOPE_TEST_CASE],
ids=lambda case: case.id,
)
def test_chat_completions_response_format_anthropic(

22
tests/integration/translation/chat_completions/streaming/test_chat_completions_streaming_anthropic.py
View file

@ -6,7 +6,7 @@ from integration._support.client import Gateway
from integration._support.provider import SharedProvider
from integration.translation.case import SSEFrame, StreamingTranslationTestCase
from integration.translation.chat_completions.bases.anthropic import (
CLAUDE_HAIKU_5_5_TEST_CASE,
CLAUDE_OPUS_4_8_TEST_CASE,
CLAUDE_SONNET_5_5_TEST_CASE,
)
from integration.translation.runner import assert_streaming_translation
@ -158,27 +158,27 @@ def _place_chunk(delta: dict[str, JsonValue]) -> SSEFrame:
"id": ANY,
"object": "chat.completion.chunk",
"created": ANY,
"model": "anthropic/claude-haiku-5-5",
"model": "anthropic/claude-opus-4-8",
"choices": [{"index": 0, "delta": delta}],
}
)
CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase(
CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationTestCase(
scenario="json_schema_cut_stream",
litellm_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.litellm_endpoint,
litellm_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.litellm_endpoint,
litellm_request={
"model": "anthropic/claude-haiku-5-5",
"model": "anthropic/claude-opus-4-8",
"max_tokens": 128,
"messages": [{"role": "user", "content": PLACE_PROMPT}],
"response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": PLACE_SCHEMA}},
"cache": {"no-cache": True},
"stream": True,
},
expected_provider_endpoint=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_endpoint,
expected_provider_headers=CLAUDE_HAIKU_5_5_TEST_CASE.expected_provider_headers,
expected_provider_endpoint=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_endpoint,
expected_provider_headers=CLAUDE_OPUS_4_8_TEST_CASE.expected_provider_headers,
expected_provider_request={
"model": "claude-haiku-5-5",
"model": "claude-opus-4-8",
"messages": [{"role": "user", "content": [{"type": "text", "text": PLACE_PROMPT}]}],
"max_tokens": 128,
"tool_choice": {"name": "json_tool_call", "type": "tool"},
@ -191,7 +191,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
data={
"type": "message_start",
"message": {
"model": "claude-haiku-5-5",
"model": "claude-opus-4-8",
"id": "msg_011CgCoP5RkexwE6N7PRZEca",
"type": "message",
"role": "assistant",
@ -263,7 +263,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
"id": ANY,
"object": "chat.completion",
"created": ANY,
"model": "anthropic/claude-haiku-5-5",
"model": "anthropic/claude-opus-4-8",
"choices": [
{
"index": 0,
@ -276,7 +276,7 @@ CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE: Final = StreamingTranslationT
@pytest.mark.parametrize(
"case",
[CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_HAIKU_5_5_JSON_SCHEMA_CUT_STREAM_TEST_CASE],
[CLAUDE_SONNET_5_5_STREAMING_TEST_CASE, CLAUDE_OPUS_4_8_JSON_SCHEMA_CUT_STREAM_TEST_CASE],
ids=lambda case: case.id,
)
def test_chat_completions_streaming_anthropic(

85
tests/unit/accounting/test_delivery_projection.py Normal file
View file

@ -0,0 +1,85 @@
import json
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import Final
from urllib.parse import unquote
from litellm.accounting.delivery_projection import entity_transactions, project_deliveries
from litellm.accounting.projection import Contribution
from litellm.repositories.accounting_delivery_repository import DeliveryProjection
_NOW: Final = datetime(2026, 11, 2, tzinfo=timezone.utc)
def _payload(spend: float, started: datetime = _NOW, window_start: datetime = _NOW) -> Mapping[str, object]:
return {
"request_id": f"request-{spend}-{started.isoformat()}",
"startTime": started,
"api_key": "key",
"user": "user/1",
"team_id": "team",
"organization_id": "org::1",
"spend": spend,
"metadata": json.dumps(
{
"accounting_entities": {
"api_key": "key",
"user": "user/1",
"team_id": "team",
"organization_id": "org::1",
},
"accounting_windows": [
{
"entity_type": "key",
"entity_id": "key",
"window_duration": "1d",
"window_start": window_start.isoformat(),
}
],
}
),
}
def _spend(contributions: tuple[Contribution, ...], table: str) -> tuple[float, ...]:
return tuple(float(dict(item.increments)["spend"]) for item in contributions if item.table == table)
def test_deliveries_for_one_destination_sum_and_keep_the_earliest_request_start() -> None:
later: Final = _NOW + timedelta(hours=1)
contributions: Final = project_deliveries(
(DeliveryProjection(_payload(0.5, started=later)), DeliveryProjection(_payload(0.25)))
)
assert _spend(contributions, "LiteLLM_VerificationToken") == (0.75,)
windows: Final = tuple(item for item in contributions if item.table == "LiteLLM_BudgetWindowSpend")
assert len(windows) == 1
assert dict(windows[0].increments) == {"spend": 0.75}
assert dict(windows[0].values)["started_at"] == str(_NOW)
def test_the_same_window_in_a_new_period_is_charged_to_its_own_row() -> None:
next_day: Final = _NOW + timedelta(days=1)
contributions: Final = project_deliveries(
(DeliveryProjection(_payload(0.25)), DeliveryProjection(_payload(0.5, window_start=next_day)))
)
windows: Final = sorted(
(dict(item.values)["window_start"], dict(item.increments)["spend"])
for item in contributions
if item.table == "LiteLLM_BudgetWindowSpend"
)
assert windows == [(_NOW.isoformat(), 0.25), (next_day.isoformat(), 0.5)]
def test_entity_transactions_key_memberships_the_way_the_writer_applies_them() -> None:
transactions: Final = entity_transactions(project_deliveries((DeliveryProjection(_payload(0.25)),)))
assert transactions["key_list_transactions"] == {"key": 0.25}
assert transactions["org_list_transactions"] == {"org::1": 0.25}
assert transactions["team_member_list_transactions"] == {"team_id::team::user_id::user/1": 0.25}
((org_member_key, org_member_spend),) = transactions["org_member_list_transactions"].items()
_, quoted_org_id, _, quoted_user_id = org_member_key.split("::")
assert (unquote(quoted_org_id), unquote(quoted_user_id), org_member_spend) == ("org::1", "user/1", 0.25)

235
tests/unit/accounting/test_measurements.py
View file

@ -1,235 +0,0 @@
from datetime import datetime, timezone
from decimal import Decimal
from typing import Final
import pytest
from litellm.accounting import (
CostEvent,
Dimension,
DimensionName,
InboundUsageEvent,
Measurement,
Metric,
MetricName,
Outcome,
OutboundUsageEvent,
PricingSnapshot,
measurements_for,
)
from litellm.types.utils import CallTypes
_START: Final = datetime(2026, 1, 1, 23, 59, 59, tzinfo=timezone.utc)
_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc)
_DIMENSIONS: Final = (
Dimension(name=DimensionName.TEAM, value="team"),
Dimension(name=DimensionName.TAG, value="alpha"),
Dimension(name=DimensionName.TAG, value="beta"),
)
def _snapshot(amount: Decimal) -> PricingSnapshot:
return PricingSnapshot(
calculator_version="test",
model="test-model",
provider=None,
pricing_source="upstream_header",
priced_model_key="test-model",
upstream_reported_cost=amount,
total_cost=amount,
)
@pytest.mark.parametrize(
("outcome", "outcome_metric"),
[
(Outcome.SUCCEEDED, MetricName.SUCCESSFUL_REQUESTS),
(Outcome.FAILED, MetricName.FAILED_REQUESTS),
(Outcome.CANCELLED, MetricName.FAILED_REQUESTS),
],
)
def test_caller_measurements_keep_both_request_and_outcome_counts(outcome, outcome_metric):
event = InboundUsageEvent(
event_id="caller-event",
request_id="caller",
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_END,
outcome=outcome,
dimensions=_DIMENSIONS,
metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)), Metric(name=outcome_metric, value=Decimal(1))),
)
assert measurements_for(event) == (
Measurement(
source_event_id="caller-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.REQUESTS, value=Decimal(1)),
),
Measurement(
source_event_id="caller-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=outcome_metric, value=Decimal(1)),
),
)
def test_usage_measurements_keep_every_metric_in_the_start_time_bucket():
event = OutboundUsageEvent(
event_id="usage-event",
attempt_id="attempt",
request_id="caller",
retry_of_attempt_id=None,
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_END,
outcome=Outcome.SUCCEEDED,
dimensions=_DIMENSIONS,
metrics=(
Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)),
Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)),
Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),
Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)),
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)),
),
)
assert measurements_for(event) == (
Measurement(
source_event_id="usage-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10)),
),
Measurement(
source_event_id="usage-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(2)),
),
Measurement(
source_event_id="usage-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),
),
Measurement(
source_event_id="usage-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.CACHE_READ_INPUT_TOKENS, value=Decimal(3)),
),
Measurement(
source_event_id="usage-event",
occurred_at=_START,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(1)),
),
)
@pytest.mark.parametrize("metrics", [None, ()])
def test_unknown_or_empty_usage_does_not_fabricate_zero_measurements(metrics):
event = OutboundUsageEvent(
event_id="usage-event",
attempt_id="attempt",
request_id=None,
retry_of_attempt_id=None,
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_END,
outcome=Outcome.SUCCEEDED,
dimensions=(),
metrics=metrics,
)
assert measurements_for(event) == ()
@pytest.mark.parametrize("amount", [Decimal(0), Decimal("0.1234567890123456789")])
def test_cost_measurements_preserve_authoritative_amount_identity_and_occurrence(amount):
event = CostEvent(
event_id="cost-event",
usage_event_id="usage-event",
occurred_at=_END,
dimensions=_DIMENSIONS,
charges=(Metric(name=MetricName.SPEND, value=amount, unit="USD"),),
snapshot=_snapshot(amount),
)
assert measurements_for(event) == (
Measurement(
source_event_id="cost-event",
occurred_at=_END,
dimensions=_DIMENSIONS,
metric=Metric(name=MetricName.SPEND, value=amount, unit="USD"),
),
)
def test_billed_retry_and_success_contribute_one_caller_request_and_both_charges():
caller = InboundUsageEvent(
event_id="caller-event",
request_id="caller",
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_END,
outcome=Outcome.SUCCEEDED,
dimensions=_DIMENSIONS,
metrics=(Metric(name=MetricName.REQUESTS, value=Decimal(1)),),
)
retry = OutboundUsageEvent(
event_id="retry-usage",
attempt_id="retry",
request_id="caller",
retry_of_attempt_id=None,
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_END,
outcome=Outcome.FAILED,
dimensions=_DIMENSIONS,
metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(4)),),
)
success = OutboundUsageEvent(
event_id="success-usage",
attempt_id="success",
request_id="caller",
retry_of_attempt_id="retry",
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_END,
outcome=Outcome.SUCCEEDED,
dimensions=_DIMENSIONS,
metrics=(Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(12)),),
)
retry_cost = CostEvent(
event_id="retry-cost",
usage_event_id="retry-usage",
occurred_at=_START,
dimensions=_DIMENSIONS,
charges=(Metric(name=MetricName.SPEND, value=Decimal("0.025"), unit="USD"),),
snapshot=_snapshot(Decimal("0.025")),
)
success_cost = CostEvent(
event_id="success-cost",
usage_event_id="success-usage",
occurred_at=_START,
dimensions=_DIMENSIONS,
charges=(Metric(name=MetricName.SPEND, value=Decimal("0.125"), unit="USD"),),
snapshot=_snapshot(Decimal("0.125")),
)
measurements = (
*measurements_for(caller),
*measurements_for(retry),
*measurements_for(success),
*measurements_for(retry_cost),
*measurements_for(success_cost),
)
assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.REQUESTS) == Decimal(1)
assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.TOTAL_TOKENS) == Decimal(16)
assert sum(m.metric.value for m in measurements if m.metric.name is MetricName.SPEND) == Decimal("0.150")
assert {m.source_event_id for m in measurements} == {
"caller-event",
"retry-usage",
"success-usage",
"retry-cost",
"success-cost",
}

28
tests/unit/accounting/test_metrics.py Normal file
View file

@ -0,0 +1,28 @@
from prometheus_client.metrics import MetricWrapperBase
from litellm.accounting import metrics
from litellm.accounting.metrics import record_deliveries
from litellm.repositories.accounting_delivery_repository import DeliveryMetrics
def _sample(metric: MetricWrapperBase, suffix: str = "") -> float:
family = metric.collect()[0]
return next(sample.value for sample in family.samples if sample.name == family.name + suffix)
def test_delivery_metrics_publish_backlog_and_clamp_clock_skew() -> None:
batches_before = _sample(metrics._BATCH_TIME, "_count")
record_deliveries(
DeliveryMetrics(pending_count=4, parked_count=1, oldest_pending_seconds=-3.0, batch_count=7, table_bytes=8192),
0.5,
)
assert (
_sample(metrics._PENDING),
_sample(metrics._PARKED),
_sample(metrics._OLDEST),
_sample(metrics._BATCH_COUNT),
_sample(metrics._TABLE_BYTES),
) == (4.0, 1.0, 0.0, 7.0, 8192.0)
assert _sample(metrics._BATCH_TIME, "_count") == batches_before + 1

126
tests/unit/accounting/test_projection.py
View file

@ -429,3 +429,129 @@ def test_guardrail_usage_is_dated_in_utc() -> None:
assert [dict(item.key)["date"] for item in projected if item.table == "LiteLLM_DailyGuardrailUsageUnits"] == [
"2026-11-03"
]
@pytest.mark.parametrize(
("untrusted", "metadata"),
[
({"request_tags": json.dumps(["x" * 16384, "normal"])}, {}),
({"end_user": "x" * 3072}, {}),
({}, {"compression_savings": {"tokens_saved": 2**63}}),
],
)
def test_untrusted_analytics_values_never_block_the_charge(
untrusted: Mapping[str, str], metadata: Mapping[str, object]
) -> None:
row: Final[Mapping[str, object]] = {
"request_id": "untrusted",
"startTime": _NOW,
"api_key": "key",
"user": "user",
"team_id": "team",
"model": "unpriced-projection-test",
"model_group": "group",
"custom_llm_provider": "openai",
"spend": 0.5,
**untrusted,
"metadata": {
**metadata,
"accounting_entities": {
"api_key": "key",
"user": "user",
"team_id": "team",
"end_user": untrusted.get("end_user"),
},
},
}
contributions: Final = project(row)
texts: Final = tuple(value for item in contributions for _, value in (*item.key, *item.values) if value)
numbers: Final = tuple(value for item in contributions for _, value in item.increments)
assert {"LiteLLM_VerificationToken", "LiteLLM_TeamTable", "LiteLLM_UserTable"} <= {
item.table for item in contributions
}
assert "LiteLLM_DailyUserSpend" in {item.table for item in contributions}
assert all(len(text.encode()) <= 2000 for text in texts)
assert all(abs(number) <= 2**40 for number in numbers)
def test_stored_payload_keeps_guardrail_billing_but_not_guardrail_content() -> None:
payload: Final = get_logging_payload(
kwargs={
"model": "projection-test",
"custom_llm_provider": "openai",
"call_type": "acompletion",
"litellm_call_id": "guardrail-request",
"litellm_params": {"metadata": {"user_api_key": "key", "user_api_key_user_id": "user"}},
},
response_obj=ModelResponse(
id="guardrail-request", model="projection-test", usage=Usage(prompt_tokens=10, completion_tokens=5)
),
start_time=_NOW,
end_time=_NOW,
)
guardrail: Final = {
"guardrail_name": "pii",
"guardrail_usage": {"requests": 1},
"guardrail_cost_by_unit": {"requests": 0.25},
"guardrail_response": "card 4111 1111 1111 1111",
"match_details": [{"matched": "4111 1111 1111 1111"}],
}
headroom: Final = {
"guardrail_name": "compress",
"guardrail_provider": "headroom",
"guardrail_response": {"tokens_saved": 7, "compressed_prompt": "card 4111 1111 1111 1111"},
}
metadata: Final = {**_FIELDS.validate_json(payload["metadata"]), "guardrail_information": [guardrail, headroom]}
accounting_input: Final[SpendLogsPayload] = {
**payload,
"spend": 0.5,
"metadata": json.dumps(metadata),
"request_tags": '["nul\x00tag"]',
}
accounting: Final = build_accounting_payload(
accounting_input, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None
)
stored: Final = json.dumps(accounting, default=str)
restored: Final = _FIELDS.validate_json(stored)
usage_rows: Final = tuple(item for item in project(restored) if item.table == "LiteLLM_DailyGuardrailUsageUnits")
assert "4111" not in stored
assert "\\u0000" not in stored
assert [dict(item.increments) for item in usage_rows] == [{"units": 1, "cost": 0.25, "untracked_units": 0}]
daily_user: Final = next(item for item in project(restored) if item.table == "LiteLLM_DailyUserSpend")
assert dict(daily_user.increments)["compression_saved_tokens"] == 7
def test_caller_supplied_savings_that_cannot_be_computed_never_drop_the_charge(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm
monkeypatch.setitem(
litellm.model_cost,
"savings-overflow-test",
{"input_cost_per_token": 0.002, "output_cost_per_token": 0.004, "litellm_provider": "openai", "mode": "chat"},
)
payload: Final[SpendLogsPayload] = {
**get_logging_payload(
kwargs={
"model": "savings-overflow-test",
"custom_llm_provider": "openai",
"call_type": "acompletion",
"litellm_call_id": "overflow-request",
"litellm_params": {"metadata": {"user_api_key": "key"}},
},
response_obj=ModelResponse(
id="overflow-request", model="savings-overflow-test", usage=Usage(prompt_tokens=10, completion_tokens=5)
),
start_time=_NOW,
end_time=_NOW,
),
"spend": 0.5,
"metadata": json.dumps({"compression_savings": {"tokens_saved": 10**400}}),
}
accounting: Final = build_accounting_payload(
payload, entities={"api_key": "key"}, model_access_groups=(), windows=(), router=lambda: None
)
restored: Final = _FIELDS.validate_json(json.dumps(accounting, default=str))
assert dict(restored["metadata"])["accounting_savings"]["compression"] == 0.0
assert [dict(item.increments)["spend"] for item in project(restored) if item.table == "LiteLLM_VerificationToken"] == [
0.5
]

482
tests/unit/accounting/test_types.py
View file

@ -1,482 +0,0 @@
import re
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from enum import Enum
from pathlib import Path
from typing import Final
import pytest
from pydantic import ValidationError
from litellm.accounting import (
AdmissionDecision,
AggregateQuery,
AggregateRow,
CostEvent,
DeliveryState,
DeliveryTarget,
Dimension,
DimensionFilter,
DimensionName,
Granularity,
InboundUsageEvent,
Metric,
MetricName,
Multiplier,
Outcome,
OutboundUsageEvent,
PricedQuantity,
PricingSnapshot,
Rate,
validate_aggregate_rows,
)
from litellm.accounting.types import PricingUnit
from litellm.types.utils import CallTypes
_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
_END: Final = datetime(2026, 1, 2, tzinfo=timezone.utc)
_REQUEST: Final = Metric(name=MetricName.REQUESTS, value=Decimal(1))
_TOKEN: Final = Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(10))
_SPEND: Final = Metric(name=MetricName.SPEND, value=Decimal("0.1234567890123456789"), unit="USD")
_TEAM: Final = Dimension(name=DimensionName.TEAM, value="team")
def _inbound(metrics=(_REQUEST,), outcome=Outcome.SUCCEEDED):
return InboundUsageEvent(
event_id="inbound:request",
request_id="request",
call_type=CallTypes.acompletion,
started_at=_START,
ended_at=_START,
outcome=outcome,
dimensions=(_TEAM,),
metrics=metrics,
)
def _outbound(metrics=(_TOKEN,), **changes):
return OutboundUsageEvent.model_validate(
{
"event_id": "outbound:attempt",
"attempt_id": "attempt",
"request_id": "request",
"retry_of_attempt_id": None,
"call_type": CallTypes.acompletion,
"started_at": _START,
"ended_at": _END,
"outcome": Outcome.SUCCEEDED,
"dimensions": (_TEAM,),
"metrics": metrics,
**changes,
}
)
def _query(**changes):
return AggregateQuery.model_validate(
{
"metric": MetricName.SPEND,
"filters": (DimensionFilter(name=DimensionName.TEAM, values=("one", "two")),),
"group_by": (DimensionName.MODEL,),
"start": _START,
"end": _END,
"granularity": Granularity.DAY,
**changes,
}
)
def test_wire_values_preserve_decimal_precision_and_normalize_aware_times():
offset = timezone(timedelta(hours=-8))
event = CostEvent(
event_id="cost:attempt",
usage_event_id="outbound:attempt",
occurred_at=datetime(2025, 12, 31, 16, tzinfo=offset),
dimensions=(_TEAM,),
charges=(_SPEND,),
snapshot=_snapshot(),
)
wire = event.model_dump(mode="json")
assert wire["occurred_at"] == "2026-01-01T00:00:00Z"
assert wire["charges"] == [{"name": "spend", "value": "0.1234567890123456789", "unit": "USD"}]
assert event.occurred_at == _START
assert CostEvent.model_validate_json(event.model_dump_json()).charges[0].value == _SPEND.value
@pytest.mark.parametrize(
("name", "value", "unit", "message"),
[
(MetricName.SPEND, Decimal(1), "count", "SPEND requires USD"),
(MetricName.REQUESTS, Decimal(1), "USD", "SPEND requires USD"),
(MetricName.PROMPT_TOKENS, Decimal("1.5"), "count", "integral"),
(MetricName.SPEND, Decimal("NaN"), "USD", "finite"),
(MetricName.SPEND, Decimal("Infinity"), "USD", "finite"),
(MetricName.SPEND, Decimal(-1), "USD", "greater than or equal"),
(MetricName.SPEND, 0.1, "USD", "instance of Decimal"),
],
)
def test_metrics_reject_wrong_units_nonfinite_values_and_binary_float_inputs(name, value, unit, message):
with pytest.raises(ValidationError, match=message):
Metric(name=name, value=value, unit=unit)
@pytest.mark.parametrize("metrics", [(_TOKEN,), (_SPEND,), ()])
def test_inbound_activity_cannot_own_usage_or_cost_or_omit_request_count(metrics):
with pytest.raises(ValidationError, match="Inbound events"):
_inbound(metrics)
@pytest.mark.parametrize(
("outcome", "metric"),
[
(Outcome.FAILED, Metric(name=MetricName.SUCCESSFUL_REQUESTS, value=Decimal(1))),
(Outcome.SUCCEEDED, Metric(name=MetricName.FAILED_REQUESTS, value=Decimal(1))),
],
)
def test_inbound_outcome_counts_cannot_contradict_outcome(outcome, metric):
with pytest.raises(ValidationError, match="count must match outcome"):
_inbound((_REQUEST, metric), outcome)
@pytest.mark.parametrize("metrics", [(_REQUEST,), (_SPEND,)])
def test_outbound_attempts_cannot_own_caller_counts_or_charges(metrics):
with pytest.raises(ValidationError, match="Outbound events own only"):
_outbound(metrics)
@pytest.mark.parametrize(
("changes", "message"),
[
({"started_at": datetime(2026, 1, 1)}, "timezone"),
({"ended_at": _START - timedelta(seconds=1)}, "End must follow start"),
({"retry_of_attempt_id": "attempt"}, "cannot retry itself"),
({"dimensions": (_TEAM, _TEAM)}, "Duplicate dimension values"),
({"dimensions": (_TEAM, Dimension(name=DimensionName.TEAM, value="other"))}, "Only tag and model access group"),
(
{
"metrics": (
Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(5)),
Metric(name=MetricName.REASONING_TOKENS, value=Decimal(6)),
)
},
"reasoning_tokens cannot exceed completion_tokens",
),
(
{
"metrics": (
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(5)),
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(6)),
)
},
"cache_creation_input_tokens_above_1hr cannot exceed cache_creation_input_tokens",
),
({"event_id": " "}, "pattern"),
({"metrics": (_TOKEN, _TOKEN)}, "Duplicate metric names"),
],
)
def test_outbound_events_reject_invalid_identity_time_and_duplicate_attribution(changes, message):
with pytest.raises(ValidationError, match=message):
_outbound(**changes)
def test_unknown_and_known_zero_usage_are_distinct_and_values_are_immutable():
unknown = _outbound(None)
zero = _outbound((Metric(name=MetricName.PROMPT_TOKENS, value=Decimal(0)),))
assert unknown.metrics is None
assert zero.metrics is not None and zero.metrics[0].value == 0
assert _inbound().started_at == _inbound().ended_at
with pytest.raises(ValidationError, match="frozen"):
zero.attempt_id = "different"
with pytest.raises(ValidationError, match="frozen"):
zero.metrics[0].value = Decimal(10)
@pytest.mark.parametrize("charges", [(), (_TOKEN,), (_SPEND, _SPEND)])
def test_cost_events_require_one_authoritative_spend(charges):
with pytest.raises(ValidationError, match=r"authoritative USD spend|Duplicate metric"):
CostEvent(
event_id="cost",
usage_event_id="usage",
occurred_at=_START,
dimensions=(),
charges=charges,
snapshot=_snapshot(),
)
def test_cost_event_cannot_reuse_its_usage_identity():
with pytest.raises(ValidationError, match="distinct identities"):
CostEvent(
event_id="usage",
usage_event_id="usage",
occurred_at=_START,
dimensions=(),
charges=(_SPEND,),
snapshot=_snapshot(),
)
@pytest.mark.parametrize("granularity", [Granularity.DAY, None])
def test_daily_and_whole_interval_queries_preserve_filter_and_group_semantics(granularity):
query = _query(granularity=granularity)
assert query.granularity is granularity
assert query.filters[0].values == ("one", "two")
assert query.group_by == (DimensionName.MODEL,)
assert query.start == _START and query.end == _END
@pytest.mark.parametrize(
("changes", "message"),
[
({"end": _START}, "End must follow start"),
({"start": _START + timedelta(hours=1)}, "UTC day boundaries"),
({"end": _END + timedelta(microseconds=1)}, "UTC day boundaries"),
({"granularity": "hour"}, "day"),
({"group_by": (DimensionName.TEAM, DimensionName.TEAM)}, "Duplicate group-by"),
({"filters": (DimensionFilter(name=DimensionName.TEAM, values=("one",)),) * 2}, "one IN filter"),
({"filters": ({"name": "team", "values": ()},)}, "at least 1"),
],
)
def test_queries_reject_unsupported_or_ambiguous_ranges_and_filters(changes, message):
with pytest.raises(ValidationError, match=message):
_query(**changes)
def test_rows_allow_adjacent_buckets_and_overlapping_distinct_groups():
first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM,), value=Decimal(1))
second = AggregateRow(start=_END, end=_END + timedelta(days=1), dimensions=(_TEAM,), value=Decimal(2))
other = AggregateRow(start=_START, end=_END, dimensions=(), value=Decimal(3))
rows = (second, other, first)
assert validate_aggregate_rows(rows) is rows
assert validate_aggregate_rows(()) == ()
def test_rows_reject_overlap_within_a_group_regardless_of_dimension_order():
model = Dimension(name=DimensionName.MODEL, value="model")
first = AggregateRow(start=_START, end=_END, dimensions=(_TEAM, model), value=Decimal(1))
overlapping = AggregateRow(start=_START, end=_END, dimensions=(model, _TEAM), value=Decimal(2))
with pytest.raises(ValueError, match="overlap within a dimension group"):
validate_aggregate_rows((first, overlapping))
def test_rows_reject_empty_intervals():
with pytest.raises(ValidationError, match="End must follow start"):
AggregateRow(start=_START, end=_START, dimensions=(), value=Decimal(0))
def test_admission_decision_preserves_a_denial_reason():
decision = AdmissionDecision(allowed=False, reason="Team budget exhausted")
assert not decision.allowed
assert decision.reason == "Team budget exhausted"
def _snapshot(**changes: object) -> PricingSnapshot:
return PricingSnapshot.model_validate(
{
"calculator_version": "1.80.0",
"model": "gpt-4o",
"provider": "openai",
"pricing_source": "cost_map",
"priced_model_key": "openai/gpt-4o",
"multipliers": (Multiplier(name="data_residency", value="eu", factor=Decimal("1.1")),),
"rates": (Rate(measure="input", unit="token", usd_per_unit=Decimal("0.0000025")),),
"usage": (PricedQuantity(measure="input", unit="token", value=_TOKEN.value),),
"input_cost": Decimal("0.000025"),
"total_cost": _SPEND.value,
**changes,
}
)
def test_cost_event_carries_a_snapshot_whose_total_matches_the_spend():
event = CostEvent(
event_id="cost:attempt",
usage_event_id="outbound:attempt",
occurred_at=_START,
dimensions=(_TEAM,),
charges=(_SPEND,),
snapshot=_snapshot(),
)
assert event.snapshot is not None
assert event.snapshot.total_cost == _SPEND.value
assert event.snapshot.multipliers[0].factor == Decimal("1.1")
with pytest.raises(ValidationError, match="Snapshot total must equal the authoritative spend"):
CostEvent(
event_id="cost:attempt",
usage_event_id="outbound:attempt",
occurred_at=_START,
dimensions=(_TEAM,),
charges=(_SPEND,),
snapshot=_snapshot(total_cost=Decimal("0.5")),
)
@pytest.mark.parametrize(
("changes", "message"),
(
(
{
"rates": (
Rate(measure="input", unit="token", usd_per_unit=Decimal(1)),
Rate(measure="input", unit="token", usd_per_unit=Decimal(2)),
)
},
"Duplicate rate measures and units",
),
({"usage": (_SPEND,)}, "PricedQuantity"),
({"usage": (_REQUEST,)}, "PricedQuantity"),
({"pricing_source": "guess"}, "pricing_source"),
({"multipliers": ({"name": "x", "value": "y", "factor": Decimal(0)},)}, "factor"),
({"schema_version": 2}, "schema_version"),
(
{"additional_costs": ({"name": "fee", "amount": Decimal(1)}, {"name": "fee", "amount": Decimal(2)})},
"Duplicate additional cost names",
),
),
)
def test_snapshots_reject_ambiguous_rates_wrong_usage_and_unknown_sources(changes: Mapping[str, object], message: str):
with pytest.raises(ValidationError, match=message):
_snapshot(**changes)
@pytest.mark.parametrize(
("unit", "value"),
[("token", "10"), ("second", "1.25"), ("character", "25"), ("image", "2"), ("request", "1")],
)
def test_snapshot_preserves_matching_rates_and_quantities_for_each_billing_unit(unit: PricingUnit, value: str) -> None:
quantity: Final = PricedQuantity(measure="input", unit=unit, value=Decimal(value))
rate: Final = Rate(measure="input", unit=unit, usd_per_unit=Decimal("0.003"))
amount: Final = quantity.value * rate.usd_per_unit
snapshot: Final = _snapshot(rates=(rate,), usage=(quantity,), total_cost=amount)
restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json())
assert restored == snapshot
assert restored.usage[0].value * restored.rates[0].usd_per_unit == amount
@pytest.mark.parametrize(
("usage", "message"),
[
((), "Every rate requires"),
(({"measure": "output", "unit": "token", "value": Decimal(10)},), "Every rate requires"),
(({"measure": "input", "unit": "second", "value": Decimal(10)},), "Every rate requires"),
(
({"measure": "input", "unit": "token", "value": Decimal(10)},) * 2,
"Duplicate priced quantity",
),
],
)
def test_snapshot_rejects_missing_mismatched_or_duplicate_priced_quantities(
usage: tuple[Mapping[str, object], ...], message: str
) -> None:
with pytest.raises(ValidationError, match=message):
_snapshot(usage=usage)
@pytest.mark.parametrize("unit", ["token", "character", "image", "request"])
def test_priced_counts_remain_integral(unit: PricingUnit) -> None:
with pytest.raises(ValidationError, match="integral"):
PricedQuantity(measure="input", unit=unit, value=Decimal("1.25"))
@pytest.mark.parametrize(
("value", "message"),
[
(Decimal(-1), "greater than or equal"),
(Decimal("NaN"), "finite"),
(Decimal("Infinity"), "finite"),
(1.25, "instance of Decimal"),
],
)
def test_pricing_values_reject_invalid_amounts(value: Decimal | float, message: str) -> None:
with pytest.raises(ValidationError, match=message):
PricedQuantity.model_validate({"measure": "input", "unit": "second", "value": value})
with pytest.raises(ValidationError, match=message):
Rate.model_validate({"measure": "input", "unit": "second", "usd_per_unit": value})
@pytest.mark.parametrize("snapshot_fields", [{}, {"snapshot": None}])
def test_cost_event_cannot_omit_its_derivation(snapshot_fields: Mapping[str, object]) -> None:
with pytest.raises(ValidationError, match="snapshot"):
CostEvent.model_validate(
{
"event_id": "cost",
"usage_event_id": "usage",
"occurred_at": _START,
"dimensions": (),
"charges": (_SPEND,),
**snapshot_fields,
}
)
def test_upstream_charge_snapshot_preserves_reported_cost_without_inventing_rates() -> None:
snapshot: Final = _snapshot(
pricing_source="upstream_header", rates=(), usage=(), upstream_reported_cost=_SPEND.value
)
restored: Final = PricingSnapshot.model_validate_json(snapshot.model_dump_json())
assert restored.upstream_reported_cost == restored.total_cost == _SPEND.value
assert restored.rates == restored.usage == ()
def test_subset_token_counts_round_trip_without_changing_totals() -> None:
metrics: Final = (
Metric(name=MetricName.COMPLETION_TOKENS, value=Decimal(10)),
Metric(name=MetricName.REASONING_TOKENS, value=Decimal(3)),
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS, value=Decimal(8)),
Metric(name=MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, value=Decimal(2)),
)
event: Final = _outbound(metrics=metrics)
assert OutboundUsageEvent.model_validate_json(event.model_dump_json()).metrics == metrics
@pytest.mark.parametrize(
"schema_path",
[
"schema.prisma",
"litellm/proxy/schema.prisma",
"litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
],
)
@pytest.mark.parametrize(
("enum_name", "python_enum"),
[
("AccountingDeliveryState", DeliveryState),
("AccountingDeliveryTarget", DeliveryTarget),
],
)
def test_prisma_accepts_exactly_the_public_accounting_enum_values(
schema_path: str, enum_name: str, python_enum: type[Enum]
) -> None:
schema: Final = (Path(__file__).resolve().parents[3] / schema_path).read_text()
declaration: Final = re.search(r"enum " + enum_name + r"\s*\{([^}]+)\}", schema)
assert declaration is not None, f"Missing persisted enum {enum_name}"
assert frozenset(declaration[1].split()) == frozenset(member.value for member in python_enum)
def test_one_event_carries_every_matched_model_access_group():
groups = (
Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="gpt-group"),
Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value="claude-group"),
)
assert _outbound(dimensions=(_TEAM, *groups)).dimensions == (_TEAM, *groups)
with pytest.raises(ValidationError, match="Duplicate dimension values"):
_outbound(dimensions=(groups[0], groups[0]))
def test_snapshots_reject_duplicate_multipliers_and_contradictory_token_counts():
multiplier = Multiplier(name="data_residency", value="eu", factor=Decimal("1.1"))
with pytest.raises(ValidationError, match="Duplicate multiplier names"):
_snapshot(multipliers=(multiplier, multiplier))
with pytest.raises(ValidationError, match="reasoning_tokens cannot exceed completion_tokens"):
_snapshot(
usage=(
PricedQuantity(measure="input", unit="token", value=_TOKEN.value),
PricedQuantity(measure="completion_tokens", unit="token", value=Decimal(5)),
PricedQuantity(measure="reasoning_tokens", unit="token", value=Decimal(6)),
)
)
def test_agent_live_spend_is_a_delivery_target():
assert DeliveryTarget.AGENT_SPEND.value == "LiteLLM_AgentsTable"

81
tests/unit/litellm_core_utils/test_fallback_generalizations.py
View file

@ -9,7 +9,8 @@ resolution (get_model_info) including the shipped rules in the bundled cost map.
import json
import logging
from collections.abc import Callable
import re
from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Final
@ -562,6 +563,23 @@ def test_shipped_rules_flag_unmapped_fable_as_always_on_thinking(shipped_cost_ma
assert other.get("thinking_always_on") is None
@pytest.mark.parametrize(
("model", "provider", "on_by_default"),
[
("claude-opus-5-9", "anthropic", True),
("claude-newfamily-6", "anthropic", True),
("us.anthropic.claude-sonnet-5-9", "bedrock", True),
("claude-opus-4-9", "anthropic", None),
],
)
def test_shipped_rules_flag_unmapped_claude_5_and_later_as_thinking_by_default(
shipped_cost_map, model, provider, on_by_default
):
assert model not in litellm.model_cost
info = litellm.get_model_info(model, custom_llm_provider=provider)
assert info.get("thinking_on_by_default") is on_by_default
@pytest.mark.parametrize(
"model,provider",
[
@ -915,6 +933,7 @@ def test_shipped_claude_thinking_rules_backfill_only_anthropic(shipped_cost_map)
assert info.get("supports_legacy_thinking") is None
assert info.get("max_input_tokens") is None
assert match_fill_missing_generalizations("claude-sonnet-4-6", "anthropic") == {
**_claude_baseline()["model_info"],
"supports_adaptive_thinking": True,
"supports_legacy_thinking": True,
"supports_tool_search": True,
@ -961,3 +980,63 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp
assert match_fill_missing_generalizations("claude-opus-5", "bedrock")["supports_tool_search"] is True
assert match_fill_missing_generalizations("claude-opus-5", "azure_ai")["supports_tool_search"] is True
assert match_fill_missing_generalizations("claude-opus-5", "perplexity") is None
CLAUDE_BASELINE_FILL_PROVIDERS: Final = (
"anthropic",
"azure_ai",
"bedrock",
"bedrock_converse",
"vertex_ai-anthropic_models",
)
def _claude_baseline() -> Mapping[str, object]:
return next(rule for rule in get_fallback_generalization_rules() if rule["name"] == "claude-family-baseline")
def _mapped_claude_chat_entries() -> tuple[tuple[str, Mapping[str, object]], ...]:
pattern: Final = re.compile(_claude_baseline()["pattern"], re.IGNORECASE)
return tuple(
(key, entry)
for key, entry in litellm.model_cost.items()
if isinstance(entry, dict) and entry.get("mode") == "chat" and pattern.search(key) is not None
)
def _fields_overriding_baseline(entry: Mapping[str, object], baseline: Mapping[str, object]) -> frozenset[str]:
return frozenset(field for field in entry.keys() & baseline.keys() if entry[field] != baseline[field])
@pytest.mark.parametrize("provider", CLAUDE_BASELINE_FILL_PROVIDERS)
def test_shipped_claude_baseline_fills_only_missing_keys_on_mapped_entries(shipped_cost_map, provider):
"""A mapped Claude entry from a first-party host gets each baseline key it leaves out,
while every baseline key the entry sets keeps the entry's own value."""
baseline: Final = _claude_baseline()["model_info"]
entries: Final = tuple(
(key, entry) for key, entry in _mapped_claude_chat_entries() if entry["litellm_provider"] == provider
)
assert any(baseline.keys() - entry.keys() for _, entry in entries)
assert any(_fields_overriding_baseline(entry, baseline) for _, entry in entries)
for key, entry in entries:
info = litellm.get_model_info(key)
assert info["key"] == key
assert {field: info[field] for field in baseline} == {
field: entry.get(field, value) for field, value in baseline.items()
}, key
def test_shipped_claude_baseline_leaves_other_providers_entries_unfilled(shipped_cost_map):
baseline: Final = _claude_baseline()["model_info"]
entries: Final = tuple(
(key, entry)
for key, entry in _mapped_claude_chat_entries()
if entry["litellm_provider"] not in CLAUDE_BASELINE_FILL_PROVIDERS
)
assert any("supports_parallel_function_calling" not in entry for _, entry in entries)
for key, entry in entries:
info = litellm.get_model_info(key)
missing = baseline.keys() - entry.keys()
assert {field: info.get(field) for field in missing} == dict.fromkeys(missing), key

116
tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py
View file

@ -3140,6 +3140,118 @@ def test_streamed_json_mode_truncated_output_is_passed_through_untouched():
assert "".join(contents) == '{"name": "json_tool_call", "parameters": {"tier": 3'
def _block_events(
index: int, block: dict[str, object], deltas: tuple[dict[str, object], ...]
) -> tuple[dict[str, object], ...]:
return (
{"type": "content_block_start", "index": index, "content_block": block},
*({"type": "content_block_delta", "index": index, "delta": delta} for delta in deltas),
{"type": "content_block_stop", "index": index},
)
def _message_events(*blocks: tuple[dict[str, object], tuple[dict[str, object], ...]]) -> tuple[dict[str, object], ...]:
starts_and_deltas: Final = tuple(
chain.from_iterable(_block_events(index, block, deltas) for index, (block, deltas) in enumerate(blocks))
)
return (
{
"type": "message_start",
"message": {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-opus-5",
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 1},
},
},
*starts_and_deltas,
{
"type": "message_delta",
"delta": {"stop_reason": "tool_use", "stop_sequence": None},
"usage": {"output_tokens": 40},
},
)
_THINKING_BLOCK: Final = (
{"type": "thinking", "thinking": "", "signature": ""},
({"type": "thinking_delta", "thinking": "7 * 142 = 994"}, {"type": "signature_delta", "signature": "sig"}),
)
_PREAMBLE_BLOCK: Final = (
{"type": "text", "text": ""},
({"type": "text_delta", "text": "**Working it out:** "}, {"type": "text_delta", "text": "the count is **120**"}),
)
def _json_tool_block(answer: dict[str, object]) -> tuple[dict[str, object], tuple[dict[str, object], ...]]:
encoded: Final = json.dumps(answer)
return (
{"type": "tool_use", "id": "toolu_1", "name": RESPONSE_FORMAT_TOOL_NAME, "input": {}},
tuple({"type": "input_json_delta", "partial_json": part} for part in (encoded[:5], encoded[5:])),
)
def _parse_stream(
events: tuple[dict[str, object], ...], schema: dict[str, object] | None = _STREAMED_PERSON_SCHEMA
) -> tuple[ModelResponseStream, ...]:
iterator: Final = ModelResponseIterator(
streaming_response=MagicMock(),
sync_stream=True,
json_mode=True,
response_format_schema=None if schema is None else response_format_schema(schema),
)
return tuple(iterator.chunk_parser(event) for event in events)
def _joined_content(chunks: tuple[ModelResponseStream, ...]) -> str:
return "".join(chunk.choices[0].delta.content or "" for chunk in chunks)
def test_streamed_json_mode_drops_prose_written_before_the_json_tool():
chunks: Final = _parse_stream(_message_events(_THINKING_BLOCK, _PREAMBLE_BLOCK, _json_tool_block(_STREAMED_PERSON)))
assert json.loads(_joined_content(chunks)) == _STREAMED_PERSON
assert "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) == (
"7 * 142 = 994"
)
assert all(chunk.choices[0].delta.tool_calls is None for chunk in chunks)
def test_streamed_json_mode_returns_prose_when_claude_skips_the_json_tool():
chunks: Final = _parse_stream(_message_events(_THINKING_BLOCK, _PREAMBLE_BLOCK))
assert _joined_content(chunks) == "**Working it out:** the count is **120**"
def test_streamed_json_mode_sends_prose_ahead_of_a_caller_tool_call():
caller_tool: Final = (
{"type": "tool_use", "id": "toolu_2", "name": "get_weather", "input": {}},
({"type": "input_json_delta", "partial_json": '{"city": "Paris"}'},),
)
chunks: Final = _parse_stream(_message_events(_PREAMBLE_BLOCK, caller_tool))
tool_chunk_index: Final = next(i for i, chunk in enumerate(chunks) if chunk.choices[0].delta.tool_calls)
assert _joined_content(chunks[: tool_chunk_index + 1]) == "**Working it out:** the count is **120**"
assert chunks[tool_chunk_index].choices[0].delta.tool_calls[0].function.name == "get_weather"
def test_streamed_native_output_text_is_not_held_back():
answer: Final = json.dumps(_STREAMED_PERSON)
text_block: Final = (
{"type": "text", "text": ""},
tuple({"type": "text_delta", "text": part} for part in (answer[:6], answer[6:])),
)
chunks: Final = _parse_stream(_message_events(text_block), schema=None)
assert tuple(chunk.choices[0].delta.content for chunk in chunks[2:4]) == (answer[:6], answer[6:])
def test_streamed_json_mode_tool_name_envelope_is_unwrapped_when_name_is_a_schema_property():
envelope = json.dumps({"name": RESPONSE_FORMAT_TOOL_NAME, "parameters": _STREAMED_CONTACT})
@ -3565,7 +3677,7 @@ def test_streamed_json_mode_partial_objects_that_repeat_a_field_keep_the_order_t
assert "".join(contents) == "".join(blocks)
def test_streamed_json_mode_text_after_the_answer_follows_the_whole_answer():
def test_streamed_json_mode_text_after_the_answer_is_dropped_like_the_non_streamed_reply():
answer = json.dumps(_STREAMED_RISK_OBJECT)
message_start, *_, message_delta = _json_tool_events(())
text_block = (
@ -3576,7 +3688,7 @@ def test_streamed_json_mode_text_after_the_answer_follows_the_whole_answer():
chunks = _json_mode_stream_chunks((message_start, *_json_tool_block_events(0, answer), *text_block, message_delta))
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == answer + " Done."
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == answer
def test_streamed_json_mode_envelope_whose_label_is_a_property_name_is_still_unwrapped():

85
tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -22,6 +22,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
RESPONSE_FORMAT_TOOL_NAME,
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
@ -1424,6 +1425,33 @@ def test_dict_fields_keep_native_output_when_the_json_tool_would_not_be_forced(m
assert "json_tool_call" not in [tool["name"] for tool in optional_params.get("tools", [])]
def test_dict_fields_take_the_unforced_json_tool_on_a_default_thinking_model(local_model_cost_map):
optional_params: Final = AnthropicConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "out",
"schema": {
"type": "object",
"properties": {"field": {"type": "object", "additionalProperties": {"type": "integer"}}},
"required": ["field"],
},
},
}
},
optional_params={},
model="claude-opus-5",
drop_params=False,
)
assert "output_format" not in optional_params
assert "tool_choice" not in optional_params
assert [
(tool["name"], tool["input_schema"]["properties"]["field"]) for tool in optional_params["tools"]
] == [("json_tool_call", {"type": "object", "additionalProperties": {"type": "integer"}})]
def test_native_structured_output_uses_bundled_capability_when_remote_map_lags(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@ -6956,6 +6984,63 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc
assert "tool_choice" not in result
@pytest.mark.parametrize(
("model", "thinking_params", "forced"),
[
("claude-opus-4-8", {"thinking": {"type": "adaptive"}}, False),
("claude-opus-5", {}, False),
("claude-sonnet-5", {}, False),
("claude-haiku-5-5", {}, False),
("claude-fable-5", {"reasoning_effort": "none"}, False),
("claude-opus-4-8", {}, True),
("claude-opus-5", {"thinking": {"type": "disabled"}}, True),
("claude-haiku-5-5", {"reasoning_effort": "none"}, True),
],
ids=[
"opus-4-8-adaptive",
"opus-5-default-thinking",
"sonnet-5-default-thinking",
"haiku-5-5-default-thinking",
"fable-5-always-on-reasoning-effort-none",
"opus-4-8-default-off",
"opus-5-thinking-disabled",
"haiku-5-5-reasoning-effort-none",
],
)
def test_response_format_tool_is_forced_only_when_the_model_will_not_think(
set_native_structured_output, model, thinking_params, forced
):
set_native_structured_output(model, None)
config: Final = AnthropicConfig()
optional_params: Final = config.map_openai_params(
non_default_params={
**thinking_params,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "capital",
"schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
},
},
optional_params={},
model=model,
drop_params=False,
)
request: Final = config.transform_request(
model=model,
messages=[{"role": "user", "content": "Name the capital of France."}],
optional_params=optional_params,
litellm_params={},
headers={},
)
assert [tool["name"] for tool in request["tools"]] == ["json_tool_call"]
assert (request.get("tool_choice") == {"name": "json_tool_call", "type": "tool"}) is forced
assert request["tools"][0].get("description") == (None if forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION)
def _eager_chat_function(**extra: object) -> dict[str, object]:
return {
"name": "write_file",

269
tests/unit/llms/azure/response/test_azure_transformation.py
View file

@ -7,15 +7,18 @@ from unittest.mock import MagicMock, patch
import httpx
import pytest
import respx
from openai.types.responses import ResponseReasoningItem
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.azure.responses.o_series_transformation import (
AzureOpenAIOSeriesResponsesAPIConfig,
)
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
from litellm.llms.azure_ai.responses.transformation import AzureAIAgentResponsesAPIConfig, AzureAIResponsesAPIConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
@ -1547,3 +1550,267 @@ def test_openai_gpt5_resolves_to_responses_config_with_temperature() -> None:
config: Final = ProviderConfigManager.get_provider_responses_api_config(provider=LlmProviders.OPENAI, model="gpt-5")
assert type(config) is OpenAIResponsesAPIConfig
assert "temperature" in config.get_supported_openai_params("gpt-5")
@pytest.mark.parametrize(
("config", "model"),
[
(AzureOpenAIResponsesAPIConfig(), "gpt-6.1-sol"),
(AzureOpenAIOSeriesResponsesAPIConfig(), "gpt-6.1-sol"),
(AzureAIResponsesAPIConfig(), "gpt-6.1-sol"),
(AzureAIAgentResponsesAPIConfig(), "agents/replay-test"),
],
ids=["azure", "o_series", "azure_ai", "azure_ai_agent"],
)
@pytest.mark.parametrize("as_model", [False, True], ids=["dict", "pydantic"])
@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "no-drop-params"])
def test_azure_reasoning_replay_removes_output_content_only_under_drop_params_without_changing_history(
config: AzureOpenAIResponsesAPIConfig, model: str, as_model: bool, drop_params: bool
) -> None:
replayable_reasoning: Final = {
"id": "rs_saved_reasoning",
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "Check the file before changing it."}],
"encrypted_content": "opaque-saved-reasoning",
}
saved_reasoning: Final = {
**replayable_reasoning,
"content": [{"type": "reasoning_text", "text": "I should inspect the file first."}],
}
tool_call: Final = {
"type": "function_call",
"id": "fc_read",
"call_id": "call_read",
"name": "read_file",
"arguments": '{"path":"main.py"}',
}
tool_result: Final = {"type": "function_call_output", "call_id": "call_read", "output": "print(1)"}
answer: Final = {
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "The file prints 1.", "annotations": []}],
}
prompt: Final = {"role": "user", "content": "Continue."}
history: Final = [
ResponseReasoningItem.model_validate(saved_reasoning) if as_model else saved_reasoning,
tool_call,
tool_result,
answer,
prompt,
]
original: Final = deepcopy(history)
request: Final = config.transform_responses_api_request(
model=model,
input=history,
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
headers={},
)
sent_reasoning: Final = replayable_reasoning if drop_params else saved_reasoning
assert request["input"] == [sent_reasoning, tool_call, tool_result, answer, prompt]
assert history == original
@pytest.mark.parametrize("as_model", [False, True], ids=["dict", "pydantic"])
def test_openai_replay_without_drop_params_keeps_reasoning_content_and_message_status(as_model: bool) -> None:
saved_reasoning: Final = {
"id": "rs_saved_reasoning",
"type": "reasoning",
"summary": [],
"content": [{"type": "reasoning_text", "text": "Inspect the file."}],
"encrypted_content": "opaque-saved-reasoning",
}
saved_message: Final = {
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "The file is ready.", "annotations": []}],
}
history: Final = [
ResponseReasoningItem.model_validate(saved_reasoning) if as_model else saved_reasoning,
saved_message,
]
original: Final = deepcopy(history)
request: Final = OpenAIResponsesAPIConfig().transform_responses_api_request(
model="gpt-6.1-sol",
input=history,
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert request["input"] == [saved_reasoning, saved_message]
assert history == original
@pytest.mark.parametrize(
"config",
[AzureOpenAIResponsesAPIConfig(), AzureOpenAIOSeriesResponsesAPIConfig(), AzureAIResponsesAPIConfig()],
ids=["azure", "o_series", "azure_ai"],
)
def test_azure_family_forwards_input_items_that_are_not_objects_for_the_provider_to_judge(
config: AzureOpenAIResponsesAPIConfig,
) -> None:
saved_reasoning: Final = {"type": "reasoning", "summary": [], "status": "completed", "encrypted_content": "opaque"}
request: Final = config.transform_responses_api_request(
model="gpt-6.1-sol",
input=["Continue.", saved_reasoning, ["nested"]],
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert request["input"] == [
"Continue.",
{"type": "reasoning", "summary": [], "encrypted_content": "opaque"},
["nested"],
]
@pytest.mark.asyncio
@pytest.mark.usefixtures(httpx_transport.__name__)
@pytest.mark.parametrize("async_call", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(("reasoning_index", "include_id"), [(51, True), (10, False)], ids=["with-id", "without-id"])
async def test_azure_responses_replays_reasoning_history_without_fields_azure_rejects(
respx_mock: respx.MockRouter, stream: bool, async_call: bool, reasoning_index: int, include_id: bool
) -> None:
prefix: Final = [{"role": "user", "content": f"Earlier context {index}."} for index in range(reasoning_index)]
replayable_reasoning: Final = {
"type": "reasoning",
"summary": [],
"encrypted_content": "opaque-saved-reasoning",
**({"id": "rs_saved_reasoning"} if include_id else {}),
}
saved_reasoning: Final = {
**replayable_reasoning,
"status": "completed",
"content": [{"type": "reasoning_text", "text": "Inspect the file before editing."}],
}
answer: Final = {
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Ready to continue.", "annotations": []}],
}
prompt: Final = {"role": "user", "content": "Continue."}
replayable_followup: Final = {
"type": "reasoning",
"summary": [],
"encrypted_content": "opaque-followup-reasoning",
**({"id": "rs_followup_reasoning"} if include_id else {}),
}
saved_followup: Final = {
**replayable_followup,
"status": "completed",
"content": [{"type": "reasoning_text", "text": "Check the next file."}],
}
history: Final = [*prefix, saved_reasoning, answer, saved_followup, prompt]
original: Final = deepcopy(history)
def respond(request: httpx.Request) -> httpx.Response:
body: Final = TypeAdapter(dict[str, JsonValue]).validate_json(request.content)
items: Final = body["input"]
assert isinstance(items, list)
reasoning: Final = items[reasoning_index]
assert isinstance(reasoning, dict)
if "status" in reasoning:
return httpx.Response(
400,
json={
"error": {
"message": f"Unknown parameter: 'input[{reasoning_index}].status'.",
"type": "invalid_request_error",
"param": f"input[{reasoning_index}].status",
"code": "unknown_parameter",
}
},
)
if reasoning.get("content"):
return httpx.Response(
400,
json={
"error": {
"message": f"Invalid 'input[{reasoning_index}].content': array too long. "
"Expected an array with maximum length 0, "
"but got an array with length 1 instead.",
"type": "invalid_request_error",
"param": f"input[{reasoning_index}].content",
"code": "array_above_max_length",
}
},
)
return httpx.Response(
200,
text=_COMPLETED_RESPONSE_SSE if stream else json.dumps(_COMPLETED_RESPONSE),
headers={"content-type": "text/event-stream" if stream else "application/json"},
)
route: Final = respx_mock.post(host="example-resource.openai.azure.com").mock(side_effect=respond)
request_params: Final = {
"model": "azure/gpt-6.1-sol",
"input": history,
"api_base": _AZURE_RESOURCE,
"api_key": "azure-test-key",
"stream": stream,
"store": False,
"include": ["reasoning.encrypted_content"],
"drop_params": True,
}
response: Final = await litellm.aresponses(**request_params) if async_call else litellm.responses(**request_params)
deltas: Final = (
(
[event.delta async for event in response if event.type == "response.output_text.delta"]
if async_call
else [event.delta for event in response if event.type == "response.output_text.delta"]
)
if stream
else []
)
text: Final = "".join(deltas) if stream else response.output[0].content[0].text
assert text == "Done."
assert route.call_count == 1
assert TypeAdapter(dict[str, JsonValue]).validate_json(route.calls[0].request.content)["input"] == [
*prefix,
replayable_reasoning,
answer,
replayable_followup,
prompt,
]
assert history == original
@pytest.mark.parametrize(
("as_model", "include_id"),
[(False, True), (False, False), (True, True)],
ids=["dictionary-with-id", "dictionary-without-id", "pydantic"],
)
@pytest.mark.parametrize("drop_params", [True, False], ids=["drop-params", "no-drop-params"])
def test_azure_reasoning_replay_drops_status_always_and_content_under_drop_params(
as_model: bool, include_id: bool, drop_params: bool
) -> None:
replayable: Final = {
"type": "reasoning",
"summary": [],
"encrypted_content": "opaque-saved-reasoning",
**({"id": "rs_saved_reasoning"} if include_id else {}),
}
saved_content: Final = [{"type": "reasoning_text", "text": "Inspect the file first."}]
saved_reasoning: Final = {**replayable, "status": "completed", "content": saved_content}
history: Final = [ResponseReasoningItem.model_validate(saved_reasoning) if as_model else saved_reasoning]
original: Final = deepcopy(history)
request: Final = AzureOpenAIResponsesAPIConfig().transform_responses_api_request(
model="gpt-6.1-sol",
input=history,
response_api_optional_request_params={},
litellm_params=GenericLiteLLMParams(drop_params=drop_params),
headers={},
)
assert request["input"] == [replayable if drop_params else {**replayable, "content": saved_content}]
assert history == original

35
tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py
View file

@ -562,19 +562,27 @@ def _invoke_params_for_schema(
@pytest.mark.parametrize(
("native_key", "thinking_params", "native"),
("model", "native_key", "thinking_params", "native"),
[
("legacy", {}, True),
("bedrock", {}, False),
("bedrock", {"thinking": {"type": "adaptive"}}, True),
("bedrock", {"reasoning_effort": "low"}, True),
("us.anthropic.claude-haiku-5-5", "legacy", {}, True),
("us.anthropic.claude-haiku-5-5", "bedrock", {}, True),
("us.anthropic.claude-opus-4-8", "bedrock", {}, False),
("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": {"type": "adaptive"}}, True),
("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True),
("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False),
],
ids=[
"legacy-key",
"bedrock-key-thinking-by-default",
"bedrock-key-thinking-off-by-default",
"bedrock-key-adaptive",
"bedrock-key-effort",
"bedrock-key-thinking-disabled",
],
ids=["legacy-key", "bedrock-key-no-thinking", "bedrock-key-adaptive", "bedrock-key-effort"],
)
def test_bedrock_native_output_key_applies_only_when_thinking_is_requested(
set_native_structured_output, native_key, thinking_params, native
def test_bedrock_native_output_key_applies_only_when_the_model_will_think(
set_native_structured_output, model, native_key, thinking_params, native
):
model = "us.anthropic.claude-haiku-5-5"
set_native_structured_output(model, native_key)
schema = {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]}
@ -914,9 +922,7 @@ def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_th
assert result["output_config"] == {"effort": "high"}
def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_tool(set_native_structured_output):
"""Anthropic accepts a forced tool_choice next to adaptive thinking, and dropping
it lets the model answer in prose instead of the requested JSON."""
def test_bedrock_chat_invoke_adaptive_thinking_leaves_the_response_format_tool_unforced(set_native_structured_output):
set_native_structured_output("us.anthropic.claude-opus-5", None)
result = AmazonAnthropicClaudeConfig().map_openai_params(
non_default_params={
@ -935,7 +941,8 @@ def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_
drop_params=False,
)
assert result["tool_choice"] == {"type": "tool", "name": result["tools"][0]["name"]}
assert [tool["name"] for tool in result["tools"]] == ["json_tool_call"]
assert "tool_choice" not in result
assert result["thinking"] == {"type": "adaptive"}
@ -2084,7 +2091,7 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co
(False, True, None, {"thinking": {"type": "adaptive"}}, True),
(False, False, None, {}, False),
(False, None, {"supports_bedrock_native_structured_output": False}, {}, False),
(True, True, None, {}, False),
(True, True, None, {"thinking": {"type": "disabled"}}, False),
],
ids=[
"cannot-force-tool",

173
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -15,6 +15,7 @@ import respx
import litellm
from litellm._logging import verbose_logger
from litellm import ModelResponse, Router, completion
from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
_bedrock_tools_pt,
@ -1440,18 +1441,27 @@ def test_thinking_request_adds_output_tokens_details_response_path():
assert result["additionalModelResponseFieldPaths"] == ("/usage/output_tokens_details",)
def test_request_without_thinking_omits_response_field_paths():
config = AmazonConverseConfig()
result = config._transform_request(
model="bedrock/converse/us.anthropic.claude-opus-4-7",
@pytest.mark.parametrize(
("model", "thinks_by_default"),
[
("bedrock/converse/us.anthropic.claude-opus-4-7", False),
("bedrock/converse/us.anthropic.claude-opus-5", True),
("us.anthropic.claude-haiku-5-5", True),
],
ids=["opus-4-7-thinking-off-by-default", "opus-5-thinking-on-by-default", "haiku-5-5-thinking-on-by-default"],
)
def test_request_without_thinking_asks_for_thinking_tokens_only_when_the_model_thinks_by_default(
local_model_cost_map: None, model: str, thinks_by_default: bool
):
result = AmazonConverseConfig()._transform_request(
model=model,
messages=[{"role": "user", "content": "hi"}],
optional_params={"maxTokens": 256},
litellm_params={},
headers={},
)
assert "additionalModelResponseFieldPaths" not in result
assert (result.get("additionalModelResponseFieldPaths") == ("/usage/output_tokens_details",)) is thinks_by_default
def test_transform_usage_prefers_provider_reasoning_tokens():
@ -4937,7 +4947,6 @@ def test_translate_response_format_native_output_config(monkeypatch):
model="anthropic.claude-sonnet-4-5-20250929-v1:0",
optional_params=optional_params,
non_default_params={"response_format": response_format},
is_thinking_enabled=False,
)
# Should have outputConfig, NOT tools
@ -4965,20 +4974,20 @@ def test_translate_response_format_native_output_config(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env)
CAPITAL_RESPONSE_FORMAT: Final = {
"type": "json_schema",
"json_schema": {
"name": "capital",
"schema": {
"type": "object",
"properties": {"city": {"type": "string"}, "country": {"type": "string"}},
"required": ["city", "country"],
"additionalProperties": False,
},
CAPITAL_JSON_SCHEMA: Final = {
"name": "capital",
"schema": {
"type": "object",
"properties": {"city": {"type": "string"}, "country": {"type": "string"}},
"required": ["city", "country"],
"additionalProperties": False,
},
}
CAPITAL_RESPONSE_FORMAT: Final = {"type": "json_schema", "json_schema": CAPITAL_JSON_SCHEMA}
ADAPTIVE_THINKING: Final = {"type": "adaptive"}
@ -5030,24 +5039,124 @@ def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag
assert "fake_stream" not in native_params
assert "outputConfig" not in tool_request
assert [tool["toolSpec"]["name"] for tool in tool_request["toolConfig"]["tools"]] == ["json_tool_call"]
assert tool_request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}}
assert "toolChoice" not in tool_request["toolConfig"]
assert tool_params.get("fake_stream", False) is stream
_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}}
@pytest.mark.parametrize(
("native_key", "thinking_params", "native"),
("model", "native_key", "model_info", "thinking_params", "forced"),
[
("legacy", {}, True),
("bedrock", {}, False),
("bedrock", {"thinking": ADAPTIVE_THINKING}, True),
("bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True),
("bedrock", {"reasoning_effort": "low"}, True),
("bedrock", {"reasoning_effort": "none"}, False),
("bedrock", {"thinking": {"type": "disabled"}}, False),
(
"us.anthropic.claude-opus-4-8",
"bedrock",
{"supports_bedrock_native_structured_output": False},
{"thinking": ADAPTIVE_THINKING},
False,
),
("us.anthropic.claude-opus-5", None, None, {}, False),
("global.anthropic.claude-sonnet-5", None, None, {}, False),
("us.anthropic.claude-haiku-5-5", None, None, {}, False),
("us.anthropic.claude-fable-5", None, None, {}, False),
("us.anthropic.claude-fable-5", None, None, {"reasoning_effort": "none"}, False),
("us.anthropic.claude-opus-4-8", None, None, {}, True),
("us.anthropic.claude-opus-5", None, None, {"thinking": {"type": "disabled"}}, True),
("us.anthropic.claude-haiku-5-5", None, None, {"reasoning_effort": "none"}, True),
],
ids=[
"opus-4-8-adaptive-native-output-fallback",
"opus-5-default-thinking",
"sonnet-5-default-thinking",
"haiku-5-5-default-thinking",
"fable-5-always-on",
"fable-5-always-on-reasoning-effort-none",
"opus-4-8-default-off",
"opus-5-thinking-disabled",
"haiku-5-5-reasoning-effort-none",
],
)
def test_json_tool_is_forced_only_when_the_model_will_not_think(
set_native_structured_output, model, native_key, model_info, thinking_params, forced
):
set_native_structured_output(model, native_key)
config: Final = AmazonConverseConfig()
response_format: Final = {
"type": "json_schema",
"json_schema": {**CAPITAL_JSON_SCHEMA, "description": "The capital of a country"},
}
optional_params: Final = config.map_openai_params(
non_default_params={"response_format": response_format, **thinking_params},
optional_params={},
model=model,
drop_params=False,
model_info=model_info,
)
request: Final = config._transform_request(
model=model,
messages=[{"role": "user", "content": "Name the capital of France."}],
optional_params=optional_params,
litellm_params={},
headers={},
)
assert "outputConfig" not in request
assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"]
assert (request["toolConfig"].get("toolChoice") == _JSON_TOOL_CHOICE) is forced
assert request["toolConfig"]["tools"][0]["toolSpec"]["description"] == (
"The capital of a country"
if forced
else f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} The capital of a country"
)
def test_unforced_json_tool_keeps_the_schema_description_on_a_non_claude_model(set_native_structured_output):
model: Final = "us.meta.llama3-3-70b-instruct-v1:0"
set_native_structured_output(model, None)
config: Final = AmazonConverseConfig()
optional_params: Final = config.map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {**CAPITAL_JSON_SCHEMA, "description": "The capital of a country"},
}
},
optional_params={},
model=model,
drop_params=False,
)
request: Final = config._transform_request(
model=model,
messages=[{"role": "user", "content": "Name the capital of France."}],
optional_params=optional_params,
litellm_params={},
headers={},
)
assert "toolChoice" not in request["toolConfig"]
assert [tool["toolSpec"]["description"] for tool in request["toolConfig"]["tools"]] == ["The capital of a country"]
@pytest.mark.parametrize(
("model", "native_key", "thinking_params", "native"),
[
("us.anthropic.claude-haiku-5-5", "legacy", {}, True),
("us.anthropic.claude-haiku-5-5", "bedrock", {}, True),
("us.anthropic.claude-opus-4-8", "bedrock", {}, False),
("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": ADAPTIVE_THINKING}, True),
("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "enabled", "budget_tokens": 1024}}, True),
("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True),
("us.anthropic.claude-haiku-5-5", "bedrock", {"reasoning_effort": "none"}, False),
("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False),
],
ids=[
"legacy-key",
"bedrock-key-no-thinking",
"bedrock-key-thinking-by-default",
"bedrock-key-thinking-off-by-default",
"bedrock-key-adaptive",
"bedrock-key-enabled",
"bedrock-key-effort",
@ -5055,10 +5164,9 @@ def test_json_schema_shape_follows_each_inference_profile_own_native_output_flag
"bedrock-key-thinking-disabled",
],
)
def test_bedrock_native_output_key_applies_only_when_thinking_is_requested(
set_native_structured_output, native_key, thinking_params, native
def test_bedrock_native_output_key_applies_only_when_the_model_will_think(
set_native_structured_output, model, native_key, thinking_params, native
):
model: Final = "us.anthropic.claude-haiku-5-5"
set_native_structured_output(model, native_key)
_, request = _converse_request_for_json_schema(model, False, thinking_params)
@ -5078,7 +5186,7 @@ _FORCED_JSON_TOOL_CHOICE: Final = {"tool": {"name": "json_tool_call"}}
(False, True, None, {"thinking": {"type": "adaptive"}}, True, None),
(False, False, None, None, False, None),
(False, None, {"supports_bedrock_native_structured_output": False}, None, False, None),
(True, True, None, None, False, _FORCED_JSON_TOOL_CHOICE),
(True, True, None, {"thinking": {"type": "disabled"}}, False, _FORCED_JSON_TOOL_CHOICE),
],
ids=[
"cannot-force-tool",
@ -5596,7 +5704,6 @@ def test_translate_response_format_fallback_tool_call():
model="anthropic.claude-3-haiku-20240307-v1:0",
optional_params=optional_params,
non_default_params={"response_format": response_format},
is_thinking_enabled=False,
)
# Should use tool-call approach, NOT outputConfig (model doesn't support native structured outputs)
@ -5633,7 +5740,6 @@ def test_native_structured_output_no_fake_stream(monkeypatch):
model="anthropic.claude-sonnet-4-5-20250929-v1:0",
optional_params=optional_params,
non_default_params={"response_format": response_format, "stream": True},
is_thinking_enabled=False,
)
assert "outputConfig" in result
@ -5954,7 +6060,6 @@ def test_json_object_no_schema_skips_tool_injection(monkeypatch):
model="anthropic.claude-sonnet-4-5-20250929-v1:0",
optional_params=optional_params,
non_default_params=non_default_params,
is_thinking_enabled=False,
)
# Should NOT use native outputConfig (no schema provided)

102
tests/unit/llms/bedrock/chat/test_invoke_handler.py
View file

@ -1149,6 +1149,108 @@ async def test_converse_streaming_completion_yields_text_and_finish_reason(
_assert_converse_stream_text_and_finish_reason(sync_chunks)
_CONVERSE_PROSE: Final = "Let me work this out. "
async def _stream_chunks(
request: Mapping[str, object], transport: httpx.MockTransport, async_mode: bool
) -> tuple[ModelResponseStream, ...]:
if not async_mode:
return tuple(litellm.completion(**request, client=HTTPHandler(client=httpx.Client(transport=transport))))
async_stream: Final = await litellm.acompletion(**request, client=AsyncHTTPHandler(transport=transport))
return tuple([chunk async for chunk in async_stream])
@pytest.mark.parametrize("async_mode", [False, True])
@pytest.mark.parametrize(
("tool_block", "stop_reason", "expected_content", "expected_tools", "expected_finish_reason"),
[
pytest.param(
{"toolUse": {"toolUseId": "tooluse_1", "name": "json_tool_call", "input": {"answer": 120}}},
"tool_use",
'{"answer": 120}',
(),
"stop",
id="prose_then_json_tool",
),
pytest.param(
{"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}},
"tool_use",
_CONVERSE_PROSE,
(("get_weather", '{"city": "Paris"}'),),
"tool_calls",
id="prose_then_caller_tool",
),
pytest.param(None, "end_turn", _CONVERSE_PROSE, (), "stop", id="json_tool_skipped"),
],
)
@pytest.mark.asyncio
async def test_converse_json_schema_stream_sends_claude_prose_only_when_claude_skips_the_json_tool(
_aws_test_credentials: None,
async_mode: bool,
tool_block: Mapping[str, object] | None,
stop_reason: str,
expected_content: str,
expected_tools: tuple[tuple[str, str], ...],
expected_finish_reason: str,
) -> None:
converse_reply: Final = {
"output": {
"message": {
"role": "assistant",
"content": [{"text": _CONVERSE_PROSE}, *((tool_block,) if tool_block is not None else ())],
}
},
"stopReason": stop_reason,
"usage": {"inputTokens": 30, "outputTokens": 20, "totalTokens": 50},
"metrics": {"latencyMs": 1},
}
def handle_request(request: httpx.Request) -> httpx.Response:
body: Final = json.loads(request.content)
assert "toolChoice" not in body["toolConfig"]
assert "json_tool_call" in [tool["toolSpec"]["name"] for tool in body["toolConfig"]["tools"]]
return httpx.Response(200, json=converse_reply)
transport: Final = httpx.MockTransport(handle_request)
request: Final = {
"model": "bedrock/us.anthropic.claude-opus-5",
"messages": [{"role": "user", "content": "What is the LCM of 24 and 120? Reply in JSON."}],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"schema": {"type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"]},
},
},
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
},
}
],
"stream": True,
}
chunks: Final = await _stream_chunks(request, transport, async_mode)
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == expected_content
tool_calls: Final = tuple(
itertools.chain.from_iterable(chunk.choices[0].delta.tool_calls or () for chunk in chunks)
)
named_calls: Final = tuple(
(call.function.name, "".join(c.function.arguments or "" for c in tool_calls))
for call in tool_calls
if call.function.name
)
assert named_calls == expected_tools
assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == [
expected_finish_reason
]
def test_mistral_invoke_stream_yields_text_and_finish_reason(
_aws_test_credentials: None,
) -> None:

45
tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py
View file

@ -19,6 +19,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION,
)
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.databricks.common_utils import DatabricksException
@ -2591,6 +2592,50 @@ def test_databricks_claude_json_schema_preserves_refs_in_tool_parameters() -> No
]
@pytest.mark.parametrize(
("model", "thinking_params", "forced"),
(
("databricks-claude-opus-5", {}, False),
("databricks-claude-opus-4-8", {"thinking": {"type": "adaptive"}}, False),
("databricks-claude-opus-5-5", {}, False),
("databricks-claude-opus-4-8", {}, True),
("databricks-claude-opus-5", {"thinking": {"type": "disabled"}}, True),
),
ids=(
"opus-5-default-thinking",
"opus-4-8-adaptive",
"opus-5-5-without-forced-tools",
"opus-4-8-default-off",
"opus-5-thinking-disabled",
),
)
def test_databricks_claude_json_tool_is_forced_only_when_the_model_will_not_think(
local_model_cost_map: None, model: str, thinking_params: dict[str, object], forced: bool
) -> None:
optional_params: Final = litellm.utils.get_optional_params(
model=model,
custom_llm_provider="databricks",
response_format=copy.deepcopy(JSON_SCHEMA_RESPONSE_FORMAT),
**thinking_params,
)
request: Final[dict[str, object]] = TypeAdapter(dict[str, object]).validate_python(
DatabricksConfig().transform_request( # pyright: ignore[reportUnknownMemberType] # inherited hook is untyped
model=model,
messages=[{"role": "user", "content": "Name the capital of France."}],
optional_params=dict(optional_params),
litellm_params={},
headers={},
)
)
assert (
request.get("tool_choice") == {"type": "function", "function": {"name": "json_tool_call"}}
) is forced
assert optional_params["tools"][0]["function"].get("description") == (
None if forced else RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
)
def test_databricks_pydantic_json_schema_preserves_nested_refs() -> None:
class Person(BaseModel):
model_config = ConfigDict(frozen=True)

1253
tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py
View file

File diff suppressed because it is too large Load diff

41
tests/unit/llms/nvidia_riva/audio_transcription/test_riva_asr_schema.py Normal file
View file

@ -0,0 +1,41 @@
from collections.abc import Iterator
from typing import Final
from google.protobuf.descriptor import FieldDescriptor
from riva.client.proto import riva_asr_pb2, riva_audio_pb2
from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema
def _declared_fields() -> Iterator[tuple[str, FieldDescriptor]]:
for message_name, _ in riva_asr_schema._MESSAGES:
message: Final = riva_asr_schema._POOL.FindMessageTypeByName(f"{riva_asr_schema._PACKAGE}.{message_name}")
yield from ((message_name, field) for field in message.fields)
def _wire_shape(field: FieldDescriptor | None) -> tuple[object, ...] | None:
if field is None:
return None
return (
field.number,
field.type,
field.label,
field.has_presence,
field.message_type.full_name if field.message_type else None,
field.enum_type.name if field.enum_type else None,
field.containing_oneof.name if field.containing_oneof else None,
)
def _nvidia_field(message_name: str, field_name: str) -> FieldDescriptor | None:
return riva_asr_pb2.DESCRIPTOR.message_types_by_name[message_name].fields_by_name.get(field_name)
def test_every_declared_field_matches_nvidias_riva_schema() -> None:
declared: Final = tuple(_declared_fields())
assert len(declared) == sum(len(fields) for _, fields in riva_asr_schema._MESSAGES)
assert {f"{message}.{field.name}": _wire_shape(field) for message, field in declared} == {
f"{message}.{field.name}": _wire_shape(_nvidia_field(message, field.name)) for message, field in declared
}
assert riva_asr_schema.LINEAR_PCM == riva_audio_pb2.AudioEncoding.LINEAR_PCM

50
tests/unit/proxy/common_utils/test_check_batch_cost.py
View file

@ -24,12 +24,12 @@ import httpx
import pytest
import respx
from fastapi import HTTPException
from litellm_enterprise.proxy.common_utils.check_batch_cost import GIVE_UP_POLLING_AFTER
import litellm
from litellm.constants import MANAGED_OBJECT_STALENESS_CUTOFF_DAYS
from litellm.types.router import LiteLLM_Params
from litellm.types.utils import LiteLLMBatch
from litellm_enterprise.proxy.common_utils.check_batch_cost import GIVE_UP_POLLING_AFTER
if TYPE_CHECKING:
from litellm.batches.batch_utils import BatchCostUsageResult
@ -254,7 +254,6 @@ class TestCheckBatchCost:
@pytest.mark.asyncio
async def test_column_absence_cached_across_cycles(self, check_batch_cost_instance, mock_prisma_client):
"""After column absence is discovered, subsequent cycles skip the primary query entirely."""
from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
# Simulate column already known absent from a previous cycle
@ -2387,8 +2386,8 @@ class TestManagedOutputFileIdEncodesPublicModelGroup:
@pytest.mark.asyncio
async def test_target_model_names_comes_from_input_file_not_provider_model(self):
from litellm.proxy.openai_files_endpoints.common_utils import (
is_base64_encoded_unified_file_id,
get_models_from_unified_file_id,
is_base64_encoded_unified_file_id,
)
output_file_id = await self._run(self._job(self._managed_input_file_id(self._PUBLIC_MODEL_GROUP)))
@ -2422,8 +2421,8 @@ class TestManagedOutputFileIdEncodesPublicModelGroup:
@pytest.mark.asyncio
async def test_falls_back_to_deployment_model_group_without_managed_input_file(self):
from litellm.proxy.openai_files_endpoints.common_utils import (
is_base64_encoded_unified_file_id,
get_models_from_unified_file_id,
is_base64_encoded_unified_file_id,
)
output_file_id = await self._run(self._job("file-raw-provider-input"))
@ -3309,7 +3308,9 @@ class TestMultiPodBatchCostClaim:
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]),
return_value=_batch_cost_result(
0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]
),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
@ -3329,15 +3330,15 @@ class TestMultiPodBatchCostClaim:
from datetime import datetime
import httpx
import litellm
import respx
import litellm
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.utils import PrismaClient
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.utils import SpecialEnums
@ -3383,6 +3384,14 @@ class TestMultiPodBatchCostClaim:
database_url="postgresql://unused:unused@localhost:5432/unused", proxy_logging_obj=dependency
)
create_spend_rows: Final = AsyncMock(return_value=1)
from litellm.proxy.db.prisma_client import PrismaWrapper
spend_prisma.db = PrismaWrapper(
original_prisma=MagicMock(
query_raw=AsyncMock(return_value=[]),
litellm_spendlogs=MagicMock(create_many=create_spend_rows),
)
)
input_rate: Final = 0.000002
litellm.register_model(
{
@ -3434,11 +3443,6 @@ class TestMultiPodBatchCostClaim:
respx.mock,
patch("litellm.proxy.proxy_server.proxy_logging_obj", dependency),
patch("litellm.proxy.proxy_server.prisma_client", spend_prisma),
patch.object(
spend_prisma.writer_db._original_prisma,
"litellm_spendlogs",
MagicMock(create_many=create_spend_rows),
),
):
respx.get(f"https://api.openai.com/v1/files/{_CLAIM_OUTPUT_FILE_ID}/content").mock(
return_value=httpx.Response(200, content=content)
@ -3483,9 +3487,9 @@ class TestMultiPodBatchCostClaim:
from datetime import datetime
import httpx
import litellm
import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.utils import LiteLLMBatch, SpecialEnums
@ -3587,9 +3591,9 @@ class TestMultiPodBatchCostClaim:
import base64
import httpx
import litellm
import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import SpecialEnums
@ -4423,9 +4427,10 @@ class TestBatchLineItemsFromThePoller:
@pytest.mark.asyncio
async def test_a_batch_bigger_than_one_part_reaches_the_callbacks_as_numbered_parts(self, monkeypatch):
import litellm
from litellm_enterprise.proxy.common_utils import check_batch_cost
import litellm
journal: list = []
_, instance = self._setup(journal)
logger = self._recording_logger(journal)
@ -4933,9 +4938,10 @@ async def test_failed_retirement_keeps_batch_available() -> None:
@pytest.mark.parametrize("resource_id", ("batch-123", "file-123", "gs://bucket/job/predictions.jsonl"))
@pytest.mark.parametrize("other_id", ("{id}4", "prefix-{id}", "{id}/child", "{id}.backup", "{id}?generation=2"))
def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other_id: str) -> None:
from litellm.exceptions import NotFoundError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
from litellm.exceptions import NotFoundError
error: Final = NotFoundError(
message=f"No object found with id '{other_id.format(id=resource_id)}'.", model="test", llm_provider="openai"
)
@ -4958,9 +4964,10 @@ def test_another_resource_in_a_404_is_not_permanent_loss(resource_id: str, other
),
)
def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, message: str) -> None:
from litellm.exceptions import NotFoundError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
from litellm.exceptions import NotFoundError
error: Final = NotFoundError(message=message.format(id=resource_id), model="test", llm_provider="openai")
assert CheckBatchCost._is_batch_gone_at_provider(error, resource_id) is True
assert CheckBatchCost._is_output_file_gone_at_provider(error, resource_id) is True
@ -4970,9 +4977,10 @@ def test_a_404_naming_the_exact_resource_is_permanent_loss(resource_id: str, mes
("missing_key", "gone"), (("out/job-123/input.jsonl.out", True), ("out/job-123/other.jsonl.out", False))
)
def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key: str, gone: bool) -> None:
from litellm.llms.bedrock.common_utils import BedrockError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
from litellm.llms.bedrock.common_utils import BedrockError
# S3 GetObject 404 body shape, checked live against a missing key on 2026-10-20
error: Final = BedrockError(
status_code=404,
@ -4985,9 +4993,10 @@ def test_an_s3_no_such_key_for_the_output_object_is_permanent_loss(missing_key:
@pytest.mark.parametrize("output_file_id", (None, ""))
def test_a_404_without_an_output_identity_remains_retryable(output_file_id: str | None) -> None:
from litellm.exceptions import NotFoundError
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
from litellm.exceptions import NotFoundError
error: Final = NotFoundError(message="Output file not found", model="test", llm_provider="openai")
assert CheckBatchCost._is_output_file_gone_at_provider(error, output_file_id) is False
@ -5003,11 +5012,12 @@ async def test_batch_cost_reaches_configured_success_callbacks(
initialized: bool,
claimed: bool,
) -> None:
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import LiteLLMBatch
from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost
events: Final[SimpleQueue[tuple[str, object, int, int]]] = SimpleQueue()

50
tests/unit/proxy/common_utils/test_model_deprecation.py
View file

@ -467,6 +467,52 @@ class TestFormatDeprecationAlertMessage:
class TestSuccessorModel:
def test_unmatched_successor_logs_warning_by_default(self, monkeypatch, caplog):
monkeypatch.setattr(litellm, "model_cost", {})
router = Router(
model_list=[
{
"model_name": "old",
"litellm_params": {"model": "openai/old-model"},
"model_info": {
"id": "1",
"deprecation_date": "2026-06-10",
"successor_model": "missing-model",
},
}
]
)
with caplog.at_level(logging.DEBUG, logger=verbose_logger.name):
collect_model_deprecations(llm_router=router, today=date(2026, 6, 1))
successor_records: Final = tuple(
record for record in caplog.records if "names successor missing-model" in record.getMessage()
)
assert len(successor_records) == 1
assert successor_records[0].levelno == logging.WARNING
def test_empty_team_public_model_name_is_normalized_to_none(self, monkeypatch):
monkeypatch.setattr(litellm, "model_cost", {})
router = Router(
model_list=[
{
"model_name": "team-model",
"litellm_params": {"model": "openai/team-model"},
"model_info": {
"id": "1",
"team_id": "team-1",
"team_public_model_name": "",
"deprecation_date": "2026-06-10",
},
}
]
)
snapshot: Final = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1))
assert snapshot.imminent[0].team_public_model_name is None
def test_should_surface_the_successor_set_on_the_deployment(self, monkeypatch):
monkeypatch.setattr(litellm, "model_cost", {})
router = Router(
@ -572,6 +618,10 @@ class TestSuccessorModel:
"typo-old": "gpt-nwe",
"wildcard-old": "anthropic/x",
}
team_deprecation: Final = next(
info for info in snapshot.imminent if info.model_name == "model_name_t1_old"
)
assert team_deprecation.team_public_model_name == "team-old"
warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()]
assert len(warnings) == 2
assert any("global-old names successor team-new" in warning for warning in warnings)

17
tests/unit/proxy/db/db_transaction_queue/test_tool_discovery_queue.py
View file

@ -2,7 +2,6 @@
Unit tests for ToolDiscoveryQueue.
"""
import pytest
from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
@ -63,3 +62,19 @@ def test_aggregate_sums_call_counts_across_workers_flushes():
{"tool_name": "get_weather", "origin": "user_defined", "call_count": 5},
{"tool_name": "search", "call_count": 1},
)
def test_requeued_rows_go_ahead_of_and_merge_with_sightings_since_the_flush(queue: ToolDiscoveryQueue) -> None:
queue.add_update({"tool_name": "get_weather", "origin": "mcp_server"})
queue.add_update({"tool_name": "search", "origin": "user_defined"})
failed_write = queue.flush()
queue.add_update({"tool_name": "translate", "origin": "user_defined"})
queue.add_update({"tool_name": "search", "origin": "user_defined"})
queue.requeue(failed_write)
assert queue.flush() == [
{"tool_name": "get_weather", "origin": "mcp_server", "call_count": 1},
{"tool_name": "search", "origin": "user_defined", "call_count": 2},
{"tool_name": "translate", "origin": "user_defined", "call_count": 1},
]

33
tests/unit/proxy/db/test_budget_window_spend_writer.py
View file

@ -407,9 +407,7 @@ async def test_roll_window_spend_row_is_conditional_on_the_stored_window_being_o
@pytest.mark.asyncio
@pytest.mark.parametrize(
("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))]
)
@pytest.mark.parametrize(("replaces_window_start", "replaced_param"), [(None, None), (WINDOW_A, datetime(2026, 8, 1))])
async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(replaces_window_start, replaced_param):
db = _FakeDB()
@ -425,7 +423,9 @@ async def test_open_window_spend_row_moves_only_the_window_the_caller_saw_end(re
((query, params),) = db.execute_raw_calls
normalized = " ".join(query.split())
assert "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE" in normalized
assert normalized.endswith("WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')")
assert normalized.endswith(
"WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
)
assert params == ("model_max_budget", "k1", "1d", datetime(2026, 8, 31), replaced_param)
@ -561,7 +561,12 @@ async def test_seed_aggregate_sql_splits_the_window_at_the_batch_start(entity_ty
((query, params),) = db.query_raw_calls
normalized = " ".join(query.split())
assert expected_column in normalized
assert "FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC'))" in normalized
assert (
'FILTER (WHERE EXISTS (SELECT 1 FROM "LiteLLM_SpendLogDelivery" delivery '
'WHERE delivery.request_id = "LiteLLM_SpendLogs".request_id) '
"OR \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC'))"
in normalized
)
assert 'FROM "LiteLLM_SpendLogs"' in normalized
# startTime is TIMESTAMP(3): the bound is floored to the second so the
# batch's own earliest row cannot round under it.
@ -635,3 +640,21 @@ async def test_rolling_a_window_row_renders_a_postgres_update_span(
new_window_start=WINDOW_B,
)
assert await postgres_span_names() == ("postgres.update LiteLLM_BudgetWindowSpend",)
@pytest.mark.asyncio
async def test_window_reads_inside_an_enclosing_transaction_use_its_connection():
client_db = _FakeDB()
transaction = _FakeDB(existing_rows=[])
await commit_window_spend_updates(
prisma_client=_FakePrismaClient(client_db),
transactions=(
build_window_spend_transaction("key", "k1", "30d", WINDOW_A, 1.0, started_at=BATCH_STARTED_AT),
),
outer_transaction=transaction,
)
assert client_db.query_raw_calls == []
assert len(transaction.query_raw_calls) == 2
assert len(transaction.batcher.calls) == 1

38
tests/unit/proxy/db/test_db_spend_update_writer.py
View file

@ -21,6 +21,7 @@ from redis.exceptions import DataError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._service_logger import ServiceTypes
from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS
from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem
@ -4382,7 +4383,6 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_only_on_rows_the_bri
assert prisma.spend_log_flush_requested.is_set() is expects_flush
@pytest.mark.asyncio
@pytest.mark.timeout(30)
async def test_a_chatgpt_responses_row_queues_without_a_default_chatgpt_login(
@ -4424,6 +4424,8 @@ def _batch_cost_payload() -> dict:
def _spend_logs_prisma(inserted: int, existing: object, taken_over: int = 1) -> MagicMock:
prisma = _tool_usage_prisma()
prisma.jsonify_object = lambda data: dict(data)
prisma.writer_db = prisma.db
prisma.db.query_raw = AsyncMock(return_value=[])
prisma.db.litellm_spendlogs.create_many = AsyncMock(return_value=inserted)
prisma.db.litellm_spendlogs.find_unique = AsyncMock(return_value=existing)
prisma.db.litellm_spendlogs.update_many = AsyncMock(return_value=taken_over)
@ -4745,6 +4747,7 @@ def _spend_logs_table_prisma() -> MagicMock:
prisma = _tool_usage_prisma()
prisma.jsonify_object = lambda data: dict(data)
prisma.db.litellm_spendlogs = _SpendLogsTable()
prisma.writer_db.query_raw = AsyncMock(return_value=[])
return prisma
@ -5839,3 +5842,36 @@ async def test_completed_spend_survives_error_log_suppression(
assert len(rows) == 1
assert rows[0]["spend"] == pytest.approx(0.23)
assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (10, 5)
@pytest.mark.parametrize("ledger_enabled", [False, True])
@pytest.mark.asyncio
async def test_request_error_counts_survive_the_ledger_path_switch(monkeypatch: pytest.MonkeyPatch, ledger_enabled: bool):
from litellm.proxy import proxy_server
from litellm.proxy.db import db_spend_update_writer as writer_module
from litellm.proxy.db.request_error_tracking import RequestErrorAccumulator
from litellm.types.proxy.request_errors import RequestErrorKey
accumulator = RequestErrorAccumulator()
prisma = _tool_usage_prisma()
prisma.get_request_status.return_value = "failure"
monkeypatch.setattr(writer_module, "request_error_accumulator", accumulator)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "general_settings", {"use_spend_logs_ledger": ledger_enabled})
monkeypatch.setattr(proxy_server, "disable_spend_logs", False)
now = datetime(2026, 11, 2, tzinfo=timezone.utc)
await DBSpendUpdateWriter().update_database(
token="key", user_id="user", team_id=None, org_id=None, end_user_id=None,
kwargs={
"model": "ledger-test", "call_type": "acompletion", "litellm_call_id": "failed-request",
"litellm_params": {"metadata": {
"user_api_key": "key", "user_api_key_user_id": "user", "status": "failure",
"error_information": {"error_code": "503"},
}},
},
completion_response=None, start_time=now, end_time=now, response_cost=0,
)
await drain_logging_tasks()
assert accumulator.drain() == {RequestErrorKey(
date="2026-11-02", api_key=prisma.spend_log_transactions[0]["api_key"], team_id="", user_id="user", model_group="ledger-test", status_code=503,
): 1}

136
tests/unit/proxy/db/test_exception_handler.py
View file

@ -22,7 +22,7 @@ from prisma.errors import (
import litellm
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, db_write_never_sent
# Test is_database_connection_error method
@ -996,3 +996,137 @@ def test_connection_closure_code_survives_prisma_data_error_wrapping() -> None:
)
assert PrismaDBExceptionHandler.is_database_transport_error(error) is True
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False
def _connector_error(message: str) -> DataError:
return DataError(
data={"user_facing_error": {"message": f"Error in connector: Error querying the database: FATAL: {message}"}}
)
def _batched_postgres_error(sqlstate: str, message: str) -> DataError:
return DataError(
data={
"user_facing_error": {
"message": "Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: "
f'None, kind: QueryError(PostgresError {{ code: "{sqlstate}", message: "{message}", severity: "FATAL" }}) }})'
}
}
)
def _localized_error(sqlstate: str) -> DataError:
return DataError(data={"user_facing_error": {"message": "FATAL: Datenbank", "meta": {"code": sqlstate}}})
_NOT_ACCEPTING_CONNECTIONS: Final = (
_connector_error("the database system is starting up"),
_connector_error("the database system is shutting down"),
_connector_error("the database system is not yet accepting connections"),
_connector_error("the database system is not accepting connections"),
_connector_error("the database system is in recovery mode"),
_localized_error("57P03"),
_batched_postgres_error("57P03", "the database system is starting up"),
EngineRequestError(
MagicMock(status=500),
'{"is_panic":false,"message":"Error in connector: Error querying the database: FATAL: the database system is '
'starting up","backtrace":null}',
),
RawQueryError(
data={
"user_facing_error": {
"message": "Raw query failed. Code: `57P03`. Message: `db error: FATAL: the database system is starting up`",
"meta": {"code": "57P03", "message": "FATAL: the database system is starting up"},
"error_code": "P2010",
}
}
),
)
_SERVER_TERMINATED: Final = (
_connector_error("terminating connection due to administrator command"),
_connector_error("terminating connection due to immediate shutdown command"),
_connector_error("terminating connection because of unexpected SIGQUIT signal"),
_connector_error("terminating connection because of crash of another server process"),
_connector_error("terminating connection due to idle-session timeout"),
_localized_error("57P01"),
_localized_error("57P02"),
_localized_error("57P05"),
_batched_postgres_error("57P01", "terminating connection due to administrator command"),
)
@pytest.mark.parametrize(
("error", "connection_error"),
[
*((error, True) for error in _NOT_ACCEPTING_CONNECTIONS),
*((error, False) for error in _SERVER_TERMINATED),
],
)
def test_postgres_lifecycle_errors_mean_the_database_is_unavailable(error: Exception, connection_error: bool) -> None:
"""A server that is restarting, shutting down or cutting sessions is a transient outage:
auth answers 503 rather than 401, and the spend-log writer must not take the rows for poison.
Only a server refusing connections opens the ``allow_requests_on_db_unavailable`` fallback,
one terminated session does not prove the database is down."""
assert {
"lifecycle": PrismaDBExceptionHandler.is_database_lifecycle_error(error),
"service_unavailable": PrismaDBExceptionHandler.is_database_service_unavailable_error(error),
"connection_error": PrismaDBExceptionHandler.is_database_connection_error(error),
"permanent": PrismaDBExceptionHandler.is_permanent_database_fault(error),
} == {
"lifecycle": True,
"service_unavailable": True,
"connection_error": connection_error,
"permanent": False,
}
@pytest.mark.parametrize(
("error", "never_sent"),
[
*((error, True) for error in _NOT_ACCEPTING_CONNECTIONS),
*((error, False) for error in _SERVER_TERMINATED),
(DataError(data={"user_facing_error": {"message": "Can't reach database server at `db`:`5432`"}}), True),
(DataError(data={"user_facing_error": {"error_code": "P1001", "message": "unreachable"}}), True),
(_capacity_error("sorry, too many clients already"), True),
(_localized_error("53300"), True),
(
DataError(
data={
"user_facing_error": {
"error_code": "P2024",
"message": "Timed out fetching a new connection from the connection pool",
}
}
),
True,
),
(httpx.ReadTimeout("read timed out"), False),
(_localized_error("57014"), False),
],
)
def test_db_write_never_sent_only_for_failures_before_any_statement_ran(error: Exception, never_sent: bool) -> None:
"""A refused, unreachable or full server provably applied nothing, while a terminated session
or a timeout may have committed before the failure, so only the former lets a non-idempotent
write be replayed."""
assert db_write_never_sent(error) is never_sent
@pytest.mark.parametrize(
"error",
[
_localized_error("57P04"),
_localized_error("57014"),
_localized_error("23505"),
DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}),
RuntimeError("the database system is starting up"),
],
)
def test_other_failures_are_not_postgres_lifecycle_errors(error: Exception) -> None:
assert PrismaDBExceptionHandler.is_database_lifecycle_error(error) is False
assert db_write_never_sent(error) is False
def test_lifecycle_error_is_seen_through_a_wrapping_exception() -> None:
wrapped: Final = RuntimeError("spend flush failed")
wrapped.__cause__ = _connector_error("the database system is starting up")
assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True

51
tests/unit/proxy/db/test_tool_registry_writer.py
View file

@ -69,9 +69,7 @@ def _make_prisma(
prisma.db.litellm_tooltable = MagicMock()
prisma.db.litellm_tooltable.upsert = AsyncMock(return_value=upsert_return)
prisma.db.execute_raw = AsyncMock(return_value=1)
prisma.db.litellm_tooltable.find_many = AsyncMock(
return_value=find_many_rows if find_many_rows is not None else []
)
prisma.db.litellm_tooltable.find_many = AsyncMock(return_value=find_many_rows if find_many_rows is not None else [])
prisma.db.litellm_tooltable.find_unique = AsyncMock(return_value=find_unique_row)
prisma.db.litellm_verificationtoken = MagicMock()
prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=list(key_rows))
@ -227,9 +225,7 @@ async def test_get_tool_found():
result = await get_tool(prisma, "my_tool")
assert result is not None
assert result.tool_name == "my_tool"
prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(
where={"tool_name": "my_tool"}
)
prisma.db.litellm_tooltable.find_unique.assert_awaited_once_with(where={"tool_name": "my_tool"})
@pytest.mark.asyncio
@ -266,9 +262,7 @@ async def test_update_tool_policy_calls_upsert_then_get_tool():
updated_by="admin",
)
prisma = _make_prisma(find_unique_row=row)
result = await update_tool_policy(
prisma, "my_tool", updated_by="admin", input_policy="blocked"
)
result = await update_tool_policy(prisma, "my_tool", updated_by="admin", input_policy="blocked")
assert result is not None
assert result.input_policy == "blocked"
prisma.db.litellm_tooltable.upsert.assert_awaited_once()
@ -567,3 +561,42 @@ async def test_failed_tool_table_write_requeues_the_rows_only_when_they_provably
left_in_redis = [json.loads(entry) for entry in redis.lists.get(REDIS_TOOL_DISCOVERY_BUFFER_KEY, [])]
expected = [[{"tool_name": "get_weather", "origin": "user_defined", "call_count": 1}]] if requeued else []
assert left_in_redis == expected, f"requeued={requeued} but redis holds {left_in_redis}"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("failure", "requeued"),
[
(_postgres_rejection("57P03"), True),
(_postgres_rejection("53300"), True),
(_postgres_rejection("57P01"), False),
(_postgres_rejection("54000"), False),
(_postgres_rejection("23502"), False),
],
ids=[
"restarting_server_keeps_the_rows",
"full_server_keeps_the_rows",
"terminated_session_may_have_counted_so_the_rows_are_dropped",
"oversized_value_is_dropped",
"rejected_row_is_dropped",
],
)
async def test_failed_write_without_redis_keeps_the_rows_in_the_queue_only_when_they_provably_did_not_apply(
failure: Exception, requeued: bool
):
queue = ToolDiscoveryQueue()
queue.add_update({"tool_name": "get_weather", "origin": "user_defined"})
prisma_client = MagicMock()
prisma_client.db.execute_raw = AsyncMock(side_effect=failure)
await commit_tool_discoveries(
prisma_client=prisma_client,
tool_discovery_queue=queue,
redis_update_buffer=RedisUpdateBuffer(redis_cache=_InMemoryRedis()),
pod_lock_manager=PodLockManager(redis_cache=_InMemoryRedis()),
use_redis_buffer=False,
)
queue.add_update({"tool_name": "get_weather", "origin": "user_defined"})
expected = [{"tool_name": "get_weather", "origin": "user_defined", "call_count": 2 if requeued else 1}]
assert queue.flush() == expected

12
tests/unit/proxy/guardrails/test_usage_tracking.py
View file

@ -654,3 +654,15 @@ async def test_one_failing_index_statement_does_not_drop_the_others_or_the_rollu
assert prisma.db.litellm_spendlogguardrailindex.create_many.await_count == 3
assert prisma.db.litellm_dailyguardrailmetrics.upsert.await_count == len(guardrail_ids)
@pytest.mark.asyncio
async def test_disabled_log_deliveries_do_not_enable_guardrail_analytics():
prisma = _prisma()
row = _payload("lean", usage={"topicPolicyUnits": 2})
row["accounting_payload"] = {"request_id": "lean"}
row["accounting_log_disabled"] = True
await process_spend_logs_guardrail_usage(prisma, [row], pending=PendingRollups())
prisma.db.litellm_dailyguardrailmetrics.upsert.assert_not_awaited()
prisma.db.litellm_spendlogguardrailindex.create_many.assert_not_awaited()
prisma.db.litellm_dailyguardrailusageunits.upsert.assert_not_awaited()

4
tests/unit/proxy/hooks/test_proxy_hooks_init.py
View file

@ -33,7 +33,7 @@ def test_managed_vector_stores_hook_registered():
assert hook_cls.__name__ == "_PROXY_LiteLLMManagedVectorStores"
def test_isolation_module_does_not_pull_in_proxy_utils():
def test_isolation_module_does_not_pull_in_proxy_utils(monkeypatch):
"""Layering guard: litellm.llms.* must not transitively import
litellm.proxy.utils, which would reintroduce the import cycle."""
import importlib
@ -44,7 +44,7 @@ def test_isolation_module_does_not_pull_in_proxy_utils():
"litellm.proxy.management_endpoints.common_utils",
"litellm.llms.base_llm.managed_resources.isolation",
]:
sys.modules.pop(mod, None)
monkeypatch.delitem(sys.modules, mod, raising=False)
importlib.import_module("litellm.llms.base_llm.managed_resources.isolation")
assert "litellm.proxy.utils" not in sys.modules

180
tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py Normal file
View file

@ -0,0 +1,180 @@
from collections.abc import Iterator
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_state_repository, router
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.repositories.config_repository import ConfigParam, ConfigRepository
from litellm.types.proxy.auth.user_roles import LitellmUserRoles
class MemoryConfig(ConfigRepository):
def __init__(self) -> None:
self.rows: dict[str, object] = {}
async def get_param(self, param_name: str) -> ConfigParam | None:
value = self.rows.get(param_name)
return None if value is None else ConfigParam(param_name, value)
async def set_param(self, param_name: str, param_value: object) -> ConfigParam:
self.rows[param_name] = param_value
return ConfigParam(param_name, param_value)
@pytest.fixture()
def client() -> Iterator[tuple[TestClient, UserAPIKeyAuth]]:
app = FastAPI()
app.include_router(router)
user = UserAPIKeyAuth(user_id="admin-one", user_role=LitellmUserRoles.PROXY_ADMIN)
repository = MemoryConfig()
app.dependency_overrides[user_api_key_auth] = lambda: user
app.dependency_overrides[alert_state_repository] = lambda: repository
with TestClient(app) as http:
yield http, user
IDENTITY = {"model": "anthropic/claude-sonnet-4-5", "deprecation_date": "2026-11-30"}
UPDATE = {**IDENTITY, "status": "resolved", "last_seen_date": "2026-11-04", "last_seen_requests": 123}
PATH = "/alerting/model-deprecations/state"
def test_new_alert_is_open(client: tuple[TestClient, UserAPIKeyAuth]) -> None:
http, _ = client
response = http.get(PATH, params=IDENTITY)
assert response.status_code == 200
assert response.json()["status"] == "open"
def test_resolution_is_shared_and_keeps_the_usage_snapshot(client: tuple[TestClient, UserAPIKeyAuth]) -> None:
http, user = client
response = http.put(PATH, json=UPDATE)
assert response.status_code == 200
assert response.json()["updated_by"] == "admin-one"
user.user_id = "admin-two"
observed = http.get(PATH, params=IDENTITY).json()
assert observed["status"] == "resolved"
assert observed["last_seen_date"] == "2026-11-04"
assert observed["last_seen_requests"] == 123
assert observed["updated_by"] == "admin-one"
def test_reopen_does_not_change_another_models_resolution(client: tuple[TestClient, UserAPIKeyAuth]) -> None:
http, _ = client
http.put(PATH, json=UPDATE)
other = {**UPDATE, "model": "openai/gpt-4-turbo"}
http.put(PATH, json=other)
assert http.put(PATH, json={**UPDATE, "status": "open"}).json()["status"] == "open"
assert http.get(PATH, params={**IDENTITY, "model": other["model"]}).json()["status"] == "resolved"
@pytest.mark.parametrize("method", ["get", "put"])
def test_internal_users_cannot_read_or_change_global_alerts(
client: tuple[TestClient, UserAPIKeyAuth], method: str
) -> None:
http, user = client
user.user_role = LitellmUserRoles.INTERNAL_USER
response = http.get(PATH, params=IDENTITY) if method == "get" else http.put(PATH, json=UPDATE)
assert response.status_code == 403
def test_viewer_can_read_but_cannot_resolve_at_the_same_path(client: tuple[TestClient, UserAPIKeyAuth]) -> None:
http, user = client
user.user_role = LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
assert http.get(PATH, params=IDENTITY).status_code == 200
assert http.put(PATH, json=UPDATE).status_code == 403
assert http.get(PATH, params=IDENTITY).json()["status"] == "open"
@pytest.mark.parametrize(
"field,value", [("status", "unknown"), ("last_seen_requests", -1), ("deprecation_date", "invalid"), ("model", "")]
)
def test_invalid_updates_are_rejected(client: tuple[TestClient, UserAPIKeyAuth], field: str, value: object) -> None:
http, _ = client
assert http.put(PATH, json={**UPDATE, field: value}).status_code == 422
assert http.get(PATH, params=IDENTITY).json()["status"] == "open"
def test_tags_are_scoped_to_model_dates_and_the_visible_keys(client: tuple[TestClient, UserAPIKeyAuth]) -> None:
from types import SimpleNamespace
from unittest.mock import AsyncMock
from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_tag_repository
http, _ = client
grouped = AsyncMock(return_value=[{"api_key": "reporting-one", "tag": "support"}])
http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace(
table=SimpleNamespace(group_by=grouped)
)
response = http.get(
"/alerting/model-deprecations/tags",
params={
"model": "openai/gpt-4-turbo",
"api_keys": ["reporting-one", "reporting-two"],
"start_date": "2026-10-06",
"end_date": "2026-11-04",
},
)
assert response.status_code == 200
assert response.json() == [{"api_key": "reporting-one", "tag": "support"}]
assert grouped.call_args.kwargs["where"] == {
"api_key": {"in": ["reporting-one", "reporting-two"]},
"model": "openai/gpt-4-turbo",
"date": {"gte": "2026-10-06", "lte": "2026-11-04"},
"api_requests": {"gt": 0},
"tag": {"not": None},
}
@pytest.mark.parametrize(
"role,expected", [(LitellmUserRoles.INTERNAL_USER, 403), (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, 200)]
)
def test_tag_access_matches_alert_visibility(
client: tuple[TestClient, UserAPIKeyAuth], role: LitellmUserRoles, expected: int
) -> None:
from types import SimpleNamespace
from unittest.mock import AsyncMock
from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_tag_repository
http, user = client
user.user_role = role
grouped = AsyncMock(return_value=[])
http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace(
table=SimpleNamespace(group_by=grouped)
)
response = http.get(
"/alerting/model-deprecations/tags",
params={"model": "test-model", "api_keys": ["one"], "start_date": "2026-10-06", "end_date": "2026-11-04"},
)
assert response.status_code == expected
if expected == 403:
grouped.assert_not_called()
def test_tag_query_rejects_more_than_100_keys_before_database_access(client: tuple[TestClient, UserAPIKeyAuth]) -> None:
from types import SimpleNamespace
from unittest.mock import AsyncMock
from litellm.proxy.alerting_endpoints.model_deprecation_alerts import (
alert_tag_repository,
)
http, _ = client
grouped = AsyncMock(return_value=[])
http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace(
table=SimpleNamespace(group_by=grouped)
)
response = http.get(
"/alerting/model-deprecations/tags",
params={
"model": "test-model",
"api_keys": [f"reporting-{index}" for index in range(101)],
"start_date": "2026-10-07",
"end_date": "2026-11-05",
},
)
assert response.status_code == 422
grouped.assert_not_called()

7
tests/unit/proxy/spend_tracking/test_budget_reservation.py
View file

@ -588,3 +588,10 @@ async def test_group_reserves_on_its_current_deployments_after_another_vendors_d
by_own_name: Final = litellm.token_counter(model="prod", messages=_MULTILINGUAL_MESSAGES)
assert openai_only != by_own_name
assert (await _input_tokens(router, "prod"))["prod"] == by_own_name
@pytest.mark.parametrize("window", [{}, {"budget_duration": "1d"}, {"reset_at": "2026-11-03T00:00:00Z"}])
def test_fixed_budget_window_requires_duration_and_reset_boundary(window: object) -> None:
from litellm.proxy.spend_tracking.budget_reservation import fixed_budget_window
assert fixed_budget_window(window) is None

116
tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -47,7 +47,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_spend_logs_metadata,
_get_vector_store_request_for_spend_logs_payload,
is_master_key,
_redact_logged_api_key,
redact_logged_api_key,
_redact_prompt_leaks_in_error_string,
sanitize_error_information_for_spend_logs,
_sanitize_guardrail_information_for_spend_logs,
@ -4038,7 +4038,7 @@ def test_sanitize_error_information_redacts_pydantic_assignment_form(
assert REDACTED_BY_LITELM_STRING in sanitized["error_message"]
# ── _redact_logged_api_key unit tests ──────────────────────────────────────
# ── redact_logged_api_key unit tests ──────────────────────────────────────
@pytest.mark.parametrize(
@ -4067,16 +4067,16 @@ def test_sanitize_error_information_persists_no_raw_model_for_an_unknown_model_r
def test_redact_logged_api_key_none_returns_none():
assert _redact_logged_api_key(None) is None
assert redact_logged_api_key(None) is None
def test_redact_logged_api_key_empty_string_returns_none():
assert _redact_logged_api_key("") is None
assert redact_logged_api_key("") is None
def test_redact_logged_api_key_sk_key_is_hashed():
raw = "sk-9876secret"
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result == hash_token(raw)
assert result is not None
assert not result.startswith("sk-")
@ -4085,15 +4085,15 @@ def test_redact_logged_api_key_sk_key_is_hashed():
def test_redact_logged_api_key_bearer_sk_equals_sk_hash():
raw = "sk-9876secret"
result_plain = _redact_logged_api_key(raw)
result_bearer = _redact_logged_api_key(f"Bearer {raw}")
result_plain = redact_logged_api_key(raw)
result_bearer = redact_logged_api_key(f"Bearer {raw}")
assert result_bearer == result_plain
def test_redact_logged_api_key_bearer_case_insensitive():
raw = "sk-9876secret"
result_lower = _redact_logged_api_key(f"bearer {raw}")
result_upper = _redact_logged_api_key(f"BEARER {raw}")
result_lower = redact_logged_api_key(f"bearer {raw}")
result_upper = redact_logged_api_key(f"BEARER {raw}")
expected = hash_token(raw)
assert result_lower == expected
assert result_upper == expected
@ -4101,7 +4101,7 @@ def test_redact_logged_api_key_bearer_case_insensitive():
def test_redact_logged_api_key_non_sk_raw_key_is_hashed():
raw = "anthropic-raw-key-xyz"
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result is not None
assert result != raw
assert len(result) == 64
@ -4111,7 +4111,7 @@ def test_redact_logged_api_key_non_sk_raw_key_is_hashed():
def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(already_hashed, already_redacted=True)
result = redact_logged_api_key(already_hashed, already_redacted=True)
assert result == already_hashed
assert hash_token(already_hashed) != result # no double-hash
@ -4119,7 +4119,7 @@ def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag():
def test_redact_logged_api_key_sha256_without_flag_is_hashed():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(already_hashed)
result = redact_logged_api_key(already_hashed)
assert result is not None
assert result != already_hashed
assert len(result) == 64
@ -4129,7 +4129,7 @@ def test_redact_logged_api_key_sha256_without_flag_is_hashed():
def test_redact_logged_api_key_long_opaque_token_is_hashed():
raw = "x1" * 450
assert len(raw) == 900
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result is not None
assert result != raw
assert raw not in result
@ -4139,27 +4139,27 @@ def test_redact_logged_api_key_long_opaque_token_is_hashed():
def test_redact_logged_api_key_hashed_jwt_passes_through():
jwt_hash = "hashed-jwt-" + "a" * 64
result = _redact_logged_api_key(jwt_hash, already_redacted=True)
result = redact_logged_api_key(jwt_hash, already_redacted=True)
assert result == jwt_hash
def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed():
lookalike = "hashed-jwt-" + "a" * 64
result = _redact_logged_api_key(lookalike)
result = redact_logged_api_key(lookalike)
assert result == hash_token(lookalike)
assert result != lookalike
def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed():
trailing = "hashed-jwt-" + "a" * 64 + "\n"
result = _redact_logged_api_key(trailing, already_redacted=True)
result = redact_logged_api_key(trailing, already_redacted=True)
assert result == hash_token(trailing)
assert result != trailing
def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed():
short_jwt = "hashed-jwt-tooshort"
result = _redact_logged_api_key(short_jwt)
result = redact_logged_api_key(short_jwt)
assert result is not None
assert result != short_jwt
assert len(result) == 64
@ -4169,14 +4169,14 @@ def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed():
def test_redact_logged_api_key_master_key_alias_passes_through():
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True)
result = redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True)
assert result == LITELLM_PROXY_MASTER_KEY_ALIAS
def test_redact_logged_api_key_master_key_alias_without_provenance_is_hashed():
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS)
result = redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS)
assert result == hash_token(LITELLM_PROXY_MASTER_KEY_ALIAS)
assert result != LITELLM_PROXY_MASTER_KEY_ALIAS
@ -4282,9 +4282,9 @@ def test_get_logging_payload_reads_used_client_oauth_token_from_the_bucket_the_p
def test_redact_logged_api_key_bearer_only_returns_none():
# "bearer " with nothing after stripping is equivalent to no key
assert _redact_logged_api_key("bearer ") is None
assert _redact_logged_api_key("Bearer ") is None
assert _redact_logged_api_key("BEARER ") is None
assert redact_logged_api_key("bearer ") is None
assert redact_logged_api_key("Bearer ") is None
assert redact_logged_api_key("BEARER ") is None
# ── _get_spend_logs_metadata key-hash invariant tests ─────────────────────
@ -4645,7 +4645,7 @@ class TestSpendLogKeyRedaction:
def test_bearer_prefixed_sk_key_is_hashed(self):
raw = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA"
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result is not None
assert not result.startswith("Bearer")
assert not result.startswith("sk-")
@ -4653,14 +4653,14 @@ class TestSpendLogKeyRedaction:
def test_bare_sk_key_is_hashed(self):
raw = "sk-WLi4iRn4JmbVlTaYw12IOA"
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result is not None
assert not result.startswith("sk-")
assert len(result) == 64
def test_bearer_lowercase_is_handled(self):
raw = "bearer sk-WLi4iRn4JmbVlTaYw12IOA"
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result is not None
assert not result.startswith("bearer")
assert not result.startswith("sk-")
@ -4668,11 +4668,11 @@ class TestSpendLogKeyRedaction:
def test_already_hashed_key_unchanged(self):
hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab"
assert _redact_logged_api_key(hashed, already_redacted=True) == hashed
assert redact_logged_api_key(hashed, already_redacted=True) == hashed
def test_bearer_prefixed_non_sk_key_is_hashed(self):
raw = "Bearer some-other-token-format"
result = _redact_logged_api_key(raw)
result = redact_logged_api_key(raw)
assert result == hash_token("some-other-token-format")
assert result is not None
assert not result.startswith("Bearer")
@ -4680,7 +4680,7 @@ class TestSpendLogKeyRedaction:
def test_bearer_and_bare_produce_same_hash(self):
bare = "sk-WLi4iRn4JmbVlTaYw12IOA"
bearer = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA"
assert _redact_logged_api_key(bare) == _redact_logged_api_key(bearer)
assert redact_logged_api_key(bare) == redact_logged_api_key(bearer)
@patch("litellm.proxy.proxy_server.master_key", None)
@ -4777,7 +4777,7 @@ def test_get_logging_payload_keeps_internal_service_account_key_readable(service
def test_redact_logged_api_key_service_account_name_without_provenance_is_hashed():
result = _redact_logged_api_key(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME)
result = redact_logged_api_key(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME)
assert result == hash_token(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME)
@ -5889,7 +5889,7 @@ def test_get_logging_payload_empty_key_slp_none_is_empty_string_not_none_literal
assert payload["api_key"] == "", (
f"Expected empty string but got {payload['api_key']!r}; "
"dropping _redact_logged_api_key's 'or \"\"' guard would yield 'None' here"
"dropping redact_logged_api_key's 'or \"\"' guard would yield 'None' here"
)
@ -5940,7 +5940,7 @@ def test_get_logging_payload_stores_user_email_in_spend_log_metadata(user_email,
def test_redact_logged_api_key_partial_sha256_is_hashed():
partial_hex = "a" * 63
result = _redact_logged_api_key(partial_hex)
result = redact_logged_api_key(partial_hex)
assert result is not None
assert result != partial_hex
assert len(result) == 64
@ -5950,7 +5950,7 @@ def test_redact_logged_api_key_partial_sha256_is_hashed():
def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True)
result = redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True)
assert result == already_hashed
assert hash_token(already_hashed) != result
@ -5958,7 +5958,7 @@ def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag():
def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(f"Bearer {already_hashed}")
result = redact_logged_api_key(f"Bearer {already_hashed}")
assert result is not None
assert result != already_hashed
assert result == hash_token(already_hashed)
@ -6570,12 +6570,12 @@ def test_get_logging_payload_never_lands_a_raw_cli_session_token(call_type: str)
def test_redact_logged_api_key_cli_session_alias_needs_alias_provenance():
assert _redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True, key_alias=_CLI_SESSION_ALIAS) == (
assert redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True, key_alias=_CLI_SESSION_ALIAS) == (
_CLI_SESSION_ALIAS
)
assert _redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True) == hash_token(_CLI_SESSION_ALIAS)
assert _redact_logged_api_key(_CLI_SESSION_ALIAS, key_alias=_CLI_SESSION_ALIAS) == hash_token(_CLI_SESSION_ALIAS)
assert _redact_logged_api_key("alice", already_redacted=True, key_alias="alice") == hash_token("alice")
assert redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True) == hash_token(_CLI_SESSION_ALIAS)
assert redact_logged_api_key(_CLI_SESSION_ALIAS, key_alias=_CLI_SESSION_ALIAS) == hash_token(_CLI_SESSION_ALIAS)
assert redact_logged_api_key("alice", already_redacted=True, key_alias="alice") == hash_token("alice")
def test_azure_spillover_stamped_from_response_headers():
@ -7814,3 +7814,45 @@ def test_spend_payload_preserves_unknown_completion_start(completion_start: str
assert payload["completionStartTime"] == (
end if completion_start == "absent" else reported if completion_start else None
)
def test_spend_log_metadata_retention_drops_accounting_inputs_it_was_not_told_to_keep() -> None:
metadata: Final = {
"status": "success",
"accounting_savings": {
"compression": 0.4,
"prompt_caching": 0.1,
"gateway_injected_caching": 0.2,
"autorouter": 0.3,
},
"accounting_windows": [
{"entity_type": "key", "entity_id": "key", "window_duration": "1d", "window_start": "2026-11-02T00:00:00"}
],
"accounting_model_access_groups": ["group"],
"user_api_key_project_id": "project",
"usage_object": {"prompt_tokens_details": {"cached_tokens": 2}},
"compression_savings": {"tokens_saved": 3},
"guardrail_information": [{"guardrail_id": "guard", "guardrail_usage": {"text": 2}}],
"internal_call_origin": "classifier",
"billed_retry_attempt": True,
"user_api_key_alias": "discard",
}
row: Final = {
"request_id": "retained-accounting",
"spend": 0.5,
"startTime": "2026-11-02T12:00:00",
"endTime": "2026-11-02T12:00:01",
"model": "model",
"model_group": "group",
"api_key": "key",
"custom_llm_provider": "openai",
"user": "user",
"team_id": "team",
"agent_id": "agent",
"request_tags": '["tag"]',
"metadata": json.dumps(metadata),
}
stored: Final = spend_log_row_with_retained_metadata(row, SpendLogsMetadataFields(include=("usage_object",)))
parsed: Final = json.loads(str(stored["metadata"]))
assert "user_api_key_alias" not in parsed
assert parsed == {"usage_object": metadata["usage_object"], "status": metadata["status"]}

Some files were not shown because too many files have changed in this diff Show more