This commit is contained in:
joshua-berri 2026-10-30 21:13:12 -07:00 • committed by GitHub
commit a8f37b23fd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 3280 additions and 62 deletions

View file

@ -200,6 +200,7 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
unpriced: Final = _without_read_price(billed.response)
cost: Final = logging_obj.response_cost_calculator(result=unpriced)
end_time: Final = datetime.now()
logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
async_coroutine=logging_obj.async_success_handler(
result=unpriced, start_time=logging_obj.start_time, end_time=end_time

View file

@ -21,6 +21,7 @@ _T = TypeVar("_T")
# When True, suppresses async logging and billing for internal sub-calls
# (e.g., emulated file-search steps that make nested LLM calls).
is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
internal_parent_request_id: Final[ContextVar[str | None]] = ContextVar("internal_parent_request_id", default=None)
# One request prices its totals, its per-token-type lines and the rates it reports on
# separate code paths. Each reads the clock for off-peak pricing, so without a pinned
@ -40,13 +41,15 @@ _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=Fa
@contextmanager
def internal_sub_call() -> Generator[None]:
def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
token: Final = is_internal_call.set(True)
parent_token: Final = internal_parent_request_id.set(parent_request_id or internal_parent_request_id.get())
try:
yield
finally:
is_internal_call.reset(token)
internal_parent_request_id.reset(parent_token)
@contextmanager

View file

@ -3,7 +3,7 @@
The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
These inputs, including the effective cost-map rates, must survive request teardown.
Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
"""
@ -38,8 +38,8 @@ class CostCompiler(Protocol):
async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
"""Reuse the committed charge or finalize one charge and its delivery intent.
Resolve captured request-scoped context by event identity and cost-map rates
at compilation. Persist the resolved inputs in the required snapshot. Replay
Resolve captured request-scoped context and rates by event identity.
Persist the resolved inputs in the required snapshot. Replay
reuses the committed result and never consumes free allowance again.
Missing pricing is Unpriced, never an inferred zero.
"""

View file

@ -276,8 +276,8 @@ class NamedCost(BaseModel):
class PricingSnapshot(BaseModel):
"""Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
Retries reuse the committed charge and snapshot.
Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
Component sums and float tolerances belong to reconciliation, not this structural contract.
"""

View file

@ -3,9 +3,10 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from dataclasses import replace as dataclasses_replace
from enum import Enum
from itertools import groupby
from typing import Any, Final, Literal, cast
from pydantic import TypeAdapter
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -15,6 +16,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
from litellm.llms.vertex_ai.batches.transformation import (
is_native_vertex_batch_output_row,
native_vertex_batch_row_model,
native_vertex_batch_row_stats,
)
from litellm.types.llms.openai import Batch
@ -210,6 +212,7 @@ class _BatchOutputLineStats:
cache_creation_tokens: int
reasoning_tokens: int
model: str | None
usage: Usage | None
def _classify_output_line_stats(
@ -268,7 +271,7 @@ def _compute_output_line_stats(
raw_model: Final = response_body.get("model")
response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
completion_details: Final = usage.completion_tokens_details
line_prompt_cost, line_completion_cost = _output_line_cost(
line_prompt_cost, line_completion_cost = _output_line_cost_or_pending(
response_body=response_body,
usage=usage,
custom_llm_provider=custom_llm_provider,
@ -286,7 +289,15 @@ def _compute_output_line_stats(
cache_read_tokens=prompt_details["cache_hit_tokens"],
cache_creation_tokens=prompt_details["cache_creation_tokens"],
reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
model=response_model,
model=response_model or model_name,
usage=(
usage.model_copy(update=ocr_usage.model_dump(exclude_none=True))
if (ocr_usage := _ocr_usage_info_from_response_body(response_body)) is not None
and ocr_usage.model_dump(exclude_none=True)
else usage
if response_body.get("usage") is not None or usage.total_tokens > 0
else None
),
)
@ -298,6 +309,47 @@ def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> O
return OCRUsageInfo.model_validate(raw_usage_info)
def _output_line_cost_or_pending(
response_body: Mapping[str, object],
usage: Usage,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
response_model: str | None,
model_info: ModelInfo | None,
vertex_location: str | None,
) -> tuple[float, float]:
try:
return _output_line_cost(
response_body, usage, custom_llm_provider, model_name, response_model, model_info, vertex_location
)
except Exception: # noqa: BLE001 # retain measured usage when existing batch pricing fails
verbose_logger.warning("Batch line pricing unavailable; retaining usage with the existing zero-cost fallback")
return 0.0, 0.0
def _combined_reported_batch_usage(usages: tuple[Usage | None, ...]) -> Mapping[str, object] | None:
from litellm.cost_calculator import BaseTokenUsageProcessor
reported: Final = [usage for usage in usages if usage is not None]
if not reported:
return None
fields: Final = tuple(TypeAdapter(dict[str, JsonValue]).validate_python(usage.model_dump()) for usage in reported)
quantities: Final = {
key: sum(value for usage in fields if isinstance(value := usage.get(key), (int, float)))
for key in OCRUsageInfo.model_fields
if any(isinstance(usage.get(key), (int, float)) for usage in fields)
}
return {**BaseTokenUsageProcessor.combine_usage_objects(usage_objects=reported).model_dump(), **quantities}
def _batch_usage_by_model(usages: tuple[tuple[str | None, Usage | None], ...]) -> Mapping[str, object]:
ordered: Final = sorted(usages, key=lambda item: item[0] or "")
return {
model: _combined_reported_batch_usage(tuple(item[1] for item in group))
for model, group in groupby(ordered, key=lambda item: item[0] or "")
}
def _output_line_cost(
response_body: Mapping[str, object],
usage: Usage,
@ -361,6 +413,8 @@ def _aggregate_batch_cost_usage_models(
completion_tokens=sum(stats.completion_tokens for stats in line_stats),
reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
**cache_token_params,
batch_usage_by_model=_batch_usage_by_model(tuple((stats.model, stats.usage) for stats in line_stats)),
batch_unknown_usage_requests=sum(stats.usage is None for stats in line_stats),
)
batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
@ -401,17 +455,21 @@ def calculate_vertex_ai_batch_cost_and_usage(
from litellm.cost_calculator import batch_cost_calculator
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
row_stats: Final = tuple(
native_vertex_batch_row_stats(
row,
model_name,
model_info=model_info,
calculate_usage=VertexGeminiConfig.calculate_usage,
cost_calculator=batch_cost_calculator,
vertex_location=vertex_location,
observations: Final = tuple(
(
native_vertex_batch_row_model(row, model_name),
native_vertex_batch_row_stats(
row,
model_name,
model_info=model_info,
calculate_usage=VertexGeminiConfig.calculate_usage,
cost_calculator=batch_cost_calculator,
vertex_location=vertex_location,
),
)
for row in vertex_ai_batch_responses
)
row_stats: Final = tuple(stats for _, stats in observations)
priced: Final = tuple(stats for stats in row_stats if stats is not None)
total_prompt_cost: Final = sum(stats.prompt_cost for stats in priced)
total_completion_cost: Final = sum(stats.completion_cost for stats in priced)
@ -435,6 +493,10 @@ def calculate_vertex_ai_batch_cost_and_usage(
total_tokens=total_tokens,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
batch_usage_by_model=_batch_usage_by_model(
tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
),
batch_unknown_usage_requests=sum(stats is None for stats in row_stats),
),
models=(
[model_name]

View file

@ -113,7 +113,7 @@ class SpeechToCompletionBridgeHandler:
voice=voice,
)
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id):
result: Final = completion(
**request_data,
)

View file

@ -16,9 +16,10 @@ import traceback
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
from httpx import Response, Timeout
from pydantic import BaseModel, JsonValue, TypeAdapter
@ -47,6 +48,7 @@ from litellm.constants import (
KUBERNETES_POD_ROUTING_KEY,
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
PROVIDER_REQUEST_ID_HEADERS,
REALTIME_METERED_USAGE_KEY,
REDACTED_BY_LITELLM,
)
from litellm.cost_calculator import (
@ -163,6 +165,7 @@ from litellm.types.llms.openai import (
ResponseIncompleteEvent,
ResponsesAPIResponse,
)
from litellm.types.llms.vertex_ai import GenerateContentResponseBody, UsageMetadata
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.rerank import RerankResponse, get_rerank_usage
@ -266,6 +269,9 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
try:
from litellm_enterprise.enterprise_callbacks.callback_controls import (
EnterpriseCallbackControls,
@ -571,6 +577,7 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
_REDACTED_RESULT: Final = TypeAdapter(object)
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -630,7 +637,60 @@ def same_attempt_marker(first: object, second: object) -> bool:
return first is second or first == second
class UsageObservation(Enum):
KNOWN = "known"
UNKNOWN = "unknown"
ESTIMATED = "estimated"
@dataclass(frozen=True, slots=True)
class UsageCaptureObservation:
usage_json: str | None
provenance: UsageObservation
response_cost: float | None = None
@classmethod
def from_usage(
cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
) -> "UsageCaptureObservation | UsageCaptureError":
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
try:
serialized: Final = (
None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
)
except Exception as error: # noqa: BLE001 # observation failures must not interrupt legacy retry handling
return UsageCaptureError(str(error))
else:
return cls(serialized, provenance, response_cost)
class Logging(LiteLLMLoggingBaseClass):
usage_capture_callback: ClassVar[
Callable[
["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
]
| None
] = None
def capture_usage_before_logging(
self,
response: object,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
*,
freeze_only: bool = False,
) -> None:
capture: Final = type(self).usage_capture_callback
if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
capture(
self,
response,
start_time or self.start_time,
end_time or datetime.datetime.now(datetime.timezone.utc),
freeze_only,
)
global \
supabaseClient, \
promptLayerLogger, \
@ -828,6 +888,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.finalized_cost: FinalizedCost | None = None
self.prepared_cost: PreparedCost | None = None
self.cost_allowance_discount: float | None = None
self.cost_allowance_allocation: SearchAllowanceAllocation | None = None
self.cost_finalization_lock = asyncio.Lock()
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
self.billed_token_rates: BilledTokenRates | None = None
@ -884,7 +945,7 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.start_time = start_time
new_call.litellm_params = litellm_params
new_call.model_call_details = {
**self.model_call_details,
**{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
"litellm_call_id": litellm_call_id,
"start_time": start_time,
"first_api_call_start_time": None,
@ -894,6 +955,7 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.finalized_cost = None
new_call.prepared_cost = None
new_call.cost_allowance_discount = None
new_call.cost_allowance_allocation = None
new_call.cost_finalization_lock = asyncio.Lock()
new_call.deferred_success_result = None
new_call.streaming_chunks = []
@ -2424,7 +2486,7 @@ class Logging(LiteLLMLoggingBaseClass):
_is_sync_litellm_request = is_sync_litellm_request
def _is_assembled_stream_success(self, result=None) -> bool:
def _is_assembled_stream_success(self, result: object = None) -> bool:
"""Final assembled stream export (not a per-chunk success call).
Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the
@ -2459,7 +2521,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
from litellm.litellm_core_utils.thread_pool_executor import executor
if self._is_assembled_stream_success(result):
if self._is_assembled_stream_success(TypeAdapter(object).validate_python(result)):
if self.model_call_details.get("has_dispatched_final_stream_success"):
return
self.model_call_details["has_dispatched_final_stream_success"] = True
@ -2477,6 +2539,11 @@ class Logging(LiteLLMLoggingBaseClass):
)
return
self.capture_usage_before_logging(
TypeAdapter(object).validate_python(result),
TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
)
await self.async_success_handler(
result,
start_time=start_time,
@ -2503,6 +2570,17 @@ class Logging(LiteLLMLoggingBaseClass):
return self.prepared_cost
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
deferred: Final = self.deferred_success_result
observed_response: Final = _LITELLM_PARAMS.validate_python(
_extract_response_obj_and_hidden_params(deferred[0] if deferred else response, None)[0]
)
self.model_call_details["_accounting_usage_observed"] = (
UsageObservation.KNOWN
if (
self.model_call_details.get("combined_usage_object")
or StandardLoggingPayloadSetup.get_usage_as_dict(observed_response)
)
else UsageObservation.UNKNOWN
)
prepared: Final = self.prepare_success_logging(
result=deferred[0] if deferred is not None else response,
start_time=deferred[1] if deferred is not None else self.start_time,
@ -2540,17 +2618,29 @@ class Logging(LiteLLMLoggingBaseClass):
},
usage,
)
self.record_partial_usage_for_failure(usage, raw_cost)
self.record_partial_usage_for_failure(
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
)
return self.prepared_cost
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
self.finalized_cost = finalized
self.model_call_details["response_cost"] = finalized.amount
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
def record_usage_capture(self, captured: "CapturedUsage") -> None:
self.model_call_details["_accounting_capture"] = captured
def record_usage_pricing(self, pricing: object) -> None:
self.model_call_details["_accounting_pricing_snapshot"] = pricing
def record_partial_usage_for_failure(
self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
) -> None:
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
if type(self).usage_capture_callback is not None:
self.model_call_details["_accounting_usage_observed"] = observation
def record_failure_after_earlier_rounds(
self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
@ -2630,6 +2720,11 @@ class Logging(LiteLLMLoggingBaseClass):
own usage while the request's failure log still bills every attempt."""
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
self.capture_failed_attempt_usage(usage)
self.model_call_details.pop("_accounting_capture", None)
self.model_call_details.pop("_accounting_pricing_snapshot", None)
self.model_call_details.pop("_accounting_usage_observed", None)
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
@ -2639,6 +2734,53 @@ class Logging(LiteLLMLoggingBaseClass):
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
def capture_failed_attempt_usage(self, usage: Usage) -> None:
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
capture: Final = type(self).usage_capture_callback
if capture is None:
return
attempt: Final = self.attempt_scoped_copy()
attempt.prepared_cost = None
attempt.finalized_cost = None
identity: Final = (
f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
)
params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
cost: Final[object] = self.model_call_details.get("response_cost")
attempt.model_call_details = {
**attempt.model_call_details,
"litellm_call_id": identity,
"parent_request_id": self.litellm_call_id,
"_accounting_capture": None,
"_accounting_prior_captures": (),
"_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
usage,
observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
float(cost) if isinstance(cost, (int, float)) else None,
),
"litellm_params": {
**params,
"metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
"litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
},
}
captured: Final = capture(
attempt,
{"id": identity, "usage": usage.model_dump(), "status": "failed"},
self.start_time,
datetime.datetime.now(datetime.timezone.utc),
False,
)
if isinstance(captured, CapturedUsage):
self.model_call_details["_accounting_prior_captures"] = (
*self.model_call_details.get("_accounting_prior_captures", ()),
captured,
)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
self.model_call_details["continued_partial_usage"] = usage
@ -2796,6 +2938,55 @@ class Logging(LiteLLMLoggingBaseClass):
_update_completion_start_time = update_completion_start_time
def normalize_endpoint_logging_result(self, result: object) -> object:
call_type: Final[object] = self.call_type
if call_type == CallTypes.anthropic_messages.value:
return self._anthropic_messages_logged_response(result=result)
if call_type in (CallTypes.generate_content.value, CallTypes.agenerate_content.value):
return self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
if call_type in (CallTypes.asend_message.value, CallTypes.send_message.value):
return self._handle_a2a_response_logging(result=result)
return result
def usage_capture_response(self, result: object) -> object:
if self.stream and not self._is_assembled_stream_success(result):
return None
copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
measured: Final = any(
isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
for frame in frames
if frame.get("type") == "response.done"
)
if not measured:
metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
return {
"id": self.litellm_call_id,
"usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
if metered is not None
else None,
}
assembled: Final = self._get_assembled_streaming_response(
normalized, self.start_time, self.start_time, True, []
)
return normalized if assembled is None else assembled
def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
from litellm.utils import count_characters
return TypeAdapter(dict[str, JsonValue]).validate_python(
{
"custom_pricing": self.custom_pricing_for(result),
"custom_cost_per_token": self.custom_cost_per_token,
"custom_cost_per_second": self.custom_cost_per_second,
"prompt_characters": count_characters(self._prompt_for_cost_calculation())
if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
else None,
}
)
def normalize_logging_result(self, result: object) -> object:
"""
Some endpoints return a different type of result than what is expected by the logging system.
@ -2806,8 +2997,20 @@ class Logging(LiteLLMLoggingBaseClass):
combined_usage_object: Final = (
RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=result)
)
metered: Final[object] = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
captured_usage: Final = (
combined_usage_object.model_copy(
update={
"metered_usage": TypeAdapter(dict[str, JsonValue]).validate_python(
TypeAdapter(object).dump_python(metered, mode="json")
)
}
)
if metered is not None
else combined_usage_object
)
logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
usage=combined_usage_object,
usage=captured_usage,
results=result,
)
@ -3080,15 +3283,7 @@ class Logging(LiteLLMLoggingBaseClass):
else cache_hit
)
if self.call_type == CallTypes.anthropic_messages.value:
result = self._anthropic_messages_logged_response(result=result)
elif (
self.call_type == CallTypes.generate_content.value
or self.call_type == CallTypes.agenerate_content.value
):
result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
elif self.call_type == CallTypes.asend_message.value or self.call_type == CallTypes.send_message.value:
result = self._handle_a2a_response_logging(result=result)
result = self.normalize_endpoint_logging_result(result)
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@ -3441,6 +3636,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@ -3964,6 +4160,8 @@ class Logging(LiteLLMLoggingBaseClass):
cost_for_built_in_tools_cost_usd_dollar=0.0,
)
self.capture_usage_before_logging(result, start_time, end_time)
self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
@ -6844,9 +7042,9 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_usage_as_dict(
response_obj: dict | None,
response_obj: dict[str, Any] | None,
combined_usage_object: Usage | None = None,
) -> dict:
) -> dict[str, Any]:
"""
Like get_usage_from_response_obj but returns a plain dict, skipping
the Pydantic Usage construction on the hot path.
@ -6858,7 +7056,21 @@ class StandardLoggingPayloadSetup:
return _empty
_raw: Final = response_obj.get("usage", None)
if _raw is None:
return dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
google_usage: Final[object] = response_obj.get("usageMetadata")
if isinstance(google_usage, dict) and _GOOGLE_USAGE_FIELDS.intersection(google_usage):
return litellm.VertexGeminiConfig.calculate_usage(
cast( # cast-ok: retain converter-supported modalities and token_count aliases
GenerateContentResponseBody, response_obj
)
).model_dump()
reported: Final = response_obj.get("usage_info")
if isinstance(reported, BaseModel):
return reported.model_dump(exclude_none=True)
return (
{key: value for key, value in reported.items() if value is not None}
if isinstance(reported, dict)
else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
)
if isinstance(_raw, ResponseAPIUsage):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
if isinstance(_raw, dict):

View file

@ -631,6 +631,8 @@ class RealTimeStreaming:
if self.session_tools or self.tool_calls:
self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
if LiteLLMLogging.usage_capture_callback is not None:
self.logging_obj.capture_usage_before_logging(self.messages)
# Route through the bounded logging worker (per-coroutine timeout +
# concurrency cap) instead of a bare create_task, so a slow callback
# can't leave suspended tasks pinning each call's response in memory.

View file

@ -129,6 +129,17 @@ def _native_vertex_row_usage(
return calculate_usage(completion_response)
def native_vertex_batch_row_model(row: Mapping[str, object], model_name: str | None) -> str | None:
response: Final = as_json_object(row.get("response"))
model_version: Final = (
response.get("model" if is_vertex_anthropic_batch_output_response(response) else "modelVersion")
if response is not None
else None
)
deployment_model: Final = model_name if model_name and "*" not in model_name else None
return deployment_model or (model_version if isinstance(model_version, str) else model_name)
def native_vertex_batch_row_stats(
row: Mapping[str, object],
model_name: str | None,
@ -156,11 +167,7 @@ def native_vertex_batch_row_stats(
if usage is None:
return None
total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
model_version: Final = response_body.get(
"model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
)
deployment_model: Final = model_name if model_name and "*" not in model_name else None
model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
model: Final = native_vertex_batch_row_model(row, model_name)
if model is None:
verbose_logger.warning(
"vertex_ai batch output row could not be costed, so it is billed at $0 and the rest of the batch "

View file

@ -38,6 +38,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_no_callback_env_reference,
)
from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.types.agents import (
AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export
)
@ -2470,6 +2471,22 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
Documents all the fields supported by `general_settings` in config.yaml
"""
accounting_capture: UsageCaptureSettings | None = Field(
None,
description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.",
)
accounting_sync_ack: bool = Field(
False,
description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. "
"Other requests remain supported with asynchronous usage capture.",
)
accounting_sync_ack_timeout_seconds: float = Field(
20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval"
)
accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field(
"fail_closed", description="Return 503 or log the persistence failure after the response wait expires"
)
request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
None,
description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.",

View file

@ -137,6 +137,7 @@ from litellm.proxy.route_llm_request import (
route_request,
)
from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost
from litellm.proxy.spend_tracking.usage_capture import capture_proxy_response
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
ProxyLogging,
@ -3366,6 +3367,9 @@ class ProxyBaseLLMRequestProcessing:
)
),
)
await capture_proxy_response(
logging_obj, response, background=_REQUEST_FIELDS.validate_python(self.data).get("background") is True
)
except (Exception, asyncio.CancelledError) as exc:
_exception_raised = True
if logging_obj.prepared_cost is not None:

View file

@ -22,7 +22,11 @@ from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
get_parent_otel_span_from_kwargs,
)
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
UsageCaptureObservation,
UsageObservation,
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
@ -63,6 +67,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
sanitize_error_information_for_spend_logs,
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs
from litellm.proxy.utils import (
REJECTED_BEFORE_REQUEST_SETUP,
REJECTED_BEFORE_REQUEST_SETUP_KEY,
@ -344,11 +349,27 @@ def _attempt_spend_row_kwargs(
"litellm_params": litellm_params,
"combined_usage_object": attempt.usage,
"response_cost": attempt.response_cost,
"_accounting_usage_observed": UsageObservation.ESTIMATED,
"_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
attempt.usage, UsageObservation.ESTIMATED, attempt.response_cost
),
"_accounting_prior_captures": (),
}
row.pop("billed_failed_attempts", None)
row.pop("_accounting_capture", None)
call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
if isinstance(call_id, str):
row["litellm_call_id"] = f"{call_id}{suffix}"
row["parent_request_id"] = call_id
metadata["parent_request_id"] = call_id
captured_attempts: Final = request_data.get("_accounting_billed_attempts")
if isinstance(captured_attempts, tuple):
for captured in TypeAdapter(tuple[object, ...]).validate_python(captured_attempts):
if (
isinstance(captured, CapturedUsage)
and captured.payload["litellm_call_id"] == row["litellm_call_id"]
):
row["_accounting_capture"] = captured
standard_logging_object: Final = request_data.get("standard_logging_object")
if isinstance(standard_logging_object, dict):
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
@ -384,11 +405,12 @@ class _ProxyDBLogger(CustomLogger):
async def async_log_success_event(
self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
) -> None:
captured_kwargs: Final = captured_callback_kwargs(kwargs, response_obj, start_time, end_time)
if self.spend_event_producer is None or not is_offloadable_success(response_obj):
await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
await self._PROXY_track_cost_callback(captured_kwargs, response_obj, start_time, end_time)
return
event: Final = build_spend_event(
kwargs,
captured_kwargs,
response_obj,
start_time,
end_time,
@ -416,6 +438,7 @@ class _ProxyDBLogger(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
traceback_str: str | None = None,
) -> None:
self._try_capture_failure_usage(request_data, original_exception, user_api_key_dict)
completed_cost: Final = (
_str_keyed_fields(request_data.get("standard_logging_object")).get("cost_finalized") is True
)
@ -595,6 +618,63 @@ class _ProxyDBLogger(CustomLogger):
),
)
def _try_capture_failure_usage(
self,
request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer
original_exception: Exception,
auth: UserAPIKeyAuth,
) -> None:
try:
self._capture_failure_usage(request_data, original_exception, auth)
except Exception: # noqa: BLE001 # capture failure must not suppress legacy billing or reservation release
verbose_proxy_logger.exception("Usage capture failed on the request failure path")
def _capture_failure_usage(
self,
request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer
original_exception: Exception,
auth: UserAPIKeyAuth,
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.db.exception_handler import exception_chain
from litellm.proxy.proxy_server import usage_capture
from litellm.proxy.spend_tracking.usage_capture import UsageAcknowledgementError
if any(isinstance(error, UsageAcknowledgementError) for error in exception_chain(original_exception)):
return
usage_meter: Final = usage_capture.meter
if usage_meter is None:
return
logging_obj: Final = request_data.get("litellm_logging_obj")
details: Final[Mapping[str, object]] = (
_str_keyed_fields(logging_obj.model_call_details) if isinstance(logging_obj, Logging) else {}
)
params: Final = _str_keyed_fields(request_data.get("litellm_params") or details.get("litellm_params"))
failure_data: Final = {
**details,
**request_data,
"litellm_params": {
**params,
"metadata": {
**_str_keyed_fields(params.get("metadata")),
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
"status": "failure",
},
},
}
start: Final = logging_obj.start_time if isinstance(logging_obj, Logging) else request_data.get("start_time")
if not isinstance(start, datetime):
return
end: Final = datetime.now(timezone.utc)
captured: Final = tuple(
capture_callback(usage_meter, row, original_exception, start, end)
for _, row in _failure_spend_rows(failure_data, 0.0, None, self._deployment_lookup)
)
request_data["_accounting_capture"] = captured[0] # rebind-ok: share one receipt with the legacy writer
request_data["_accounting_billed_attempts"] = captured[1:] # rebind-ok: share attempt receipts
async def _write_failure_spend_rows(
self,
user_api_key_dict: UserAPIKeyAuth,

View file

@ -472,7 +472,6 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l
log_llm_api_exception,
open_sse_before_first_byte,
proxy_exception_from_route_error,
request_litellm_call_id,
resolve_litellm_call_id,
should_return_raw_model_name,
sse_error_payload,
@ -839,6 +838,8 @@ from litellm.proxy.spend_tracking.spend_event_producer import (
SpendEventProducer,
build_spend_event_producer,
)
from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.proxy.telemetry.endpoints import router as telemetry_router
from litellm.proxy.telemetry.middleware import TelemetryMiddleware
from litellm.proxy.telemetry.runtime import TelemetryRuntime
@ -1328,6 +1329,7 @@ async def _flush_spend_around_logging_drain_on_shutdown(
finally:
await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock))
await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock))
await usage_capture.stop(GracefulShutdownManager.remaining(clock=clock))
await flush_spend()
@ -3201,9 +3203,37 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False):
spend_event_producer: SpendEventProducer | None = None
usage_capture: Final = UsageCaptureRuntime()
def _configure_usage_capture() -> None:
settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
capture_settings: Final = settings.get("accounting_capture")
if settings.get("accounting_sync_ack") and capture_settings is None:
raise ValueError("accounting_sync_ack requires accounting_capture")
if capture_settings is None:
if usage_capture.meter is not None:
raise ValueError("Disabling accounting_capture requires a worker restart")
return
if (
prisma_client is None
or CollectorSettings().enabled
or os.getenv("SPEND_LOGS_URL")
or litellm.apply_search_free_tier
):
raise ValueError(
"Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier"
)
from litellm.proxy.db.accounting_ledger import AccountingLedger
ledger: Final = AccountingLedger.for_client(prisma_client)
writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch)
usage_capture.start(writer.write, UsageCaptureSettings.model_validate(capture_settings))
def cost_tracking():
global prisma_client, spend_event_producer
_configure_usage_capture()
if prisma_client is not None:
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
@ -13003,8 +13033,10 @@ async def completion(
"""
global user_temperature, user_request_timeout, user_max_tokens, user_api_base
data = {}
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
try:
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
base_llm_response_processor.data = data
if user_api_key_dict is not None:
if data.get("metadata") is None:
data["metadata"] = {}
@ -13018,7 +13050,6 @@ async def completion(
data["metadata"]["user_api_key_org_alias"] = user_api_key_dict.organization_alias
if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None:
data["metadata"]["agent_id"] = user_api_key_dict.agent_id
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
return await base_llm_response_processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
@ -13120,9 +13151,12 @@ async def completion(
return _response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=base_llm_response_processor.data,
dispatched_call_type=base_llm_response_processor.route_type,
)
litellm_call_id: Final = request_litellm_call_id(data)
litellm_call_id: Final = base_llm_response_processor.litellm_call_id
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)

View file

@ -89,6 +89,7 @@ async def _finish_finalization(
if litellm.apply_search_free_tier
else None
)
logging_obj.cost_allowance_allocation = allocation # rebind-ok: retain the allocation for durable capture
logging_obj.cost_allowance_discount = ( # rebind-ok: retain allocation across snapshot retries
allocation.discount if allocation is not None else 0.0
)

View file

@ -3,7 +3,7 @@ import json
import os
import re
import secrets
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime, timezone
from datetime import datetime as dt
from functools import reduce
@ -55,6 +55,8 @@ from litellm.litellm_core_utils.get_llm_provider_logic import declared_authentic
from litellm.litellm_core_utils.hidden_params import served_from_cache
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
UsageObservation,
coerce_model_access_groups,
is_valid_sha256_hash,
request_model_access_groups_from_litellm_params,
@ -599,6 +601,7 @@ def get_logging_payload(
start_time: datetime,
end_time: datetime,
llm_router: "Router | None" = None,
capture_usage: Callable[[SpendLogsPayload, str | None], None] | None = None,
) -> SpendLogsPayload:
if kwargs is None:
kwargs = {}
@ -622,7 +625,9 @@ def get_logging_payload(
routing_metadata,
routing_litellm_params,
)
completion_start_time: Final = kwargs.get("completion_start_time", end_time)
completion_start_time: Final = TypeAdapter[datetime | None](datetime | None).validate_python(
kwargs.get("completion_start_time", end_time)
)
call_type: Final = kwargs.get("call_type")
cache_hit: Final = kwargs.get("cache_hit", False)
@ -928,7 +933,9 @@ def get_logging_payload(
cache_hit=str(cache_hit),
startTime=_ensure_datetime_utc(start_time),
endTime=_ensure_datetime_utc(end_time),
completionStartTime=_ensure_datetime_utc(completion_start_time),
completionStartTime=(
_ensure_datetime_utc(completion_start_time) if completion_start_time is not None else None
),
model=model_name,
user=metadata.get("user_api_key_user_id", "") or "",
team_id=metadata.get("user_api_key_team_id", "") or "",
@ -984,6 +991,14 @@ def get_logging_payload(
payload.get("spend"),
)
if capture_usage is not None:
observed_usage: Final = (
StandardLoggingPayloadSetup.get_usage_as_dict({"usage": usage})
if usage and kwargs.get("_accounting_usage_observed") is not UsageObservation.UNKNOWN
else None
)
capture_usage(payload, safe_dumps(observed_usage) if observed_usage is not None else None)
# Explicitly clear large intermediate objects to reduce memory pressure
del response_obj_dict, usage, clean_metadata, additional_usage_values

View file

@ -0,0 +1,833 @@
from __future__ import annotations
import asyncio
import json
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from decimal import Decimal
from itertools import chain
from typing import TYPE_CHECKING, Final
from pydantic import BaseModel, JsonValue, TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.accounting import (
Dimension,
DimensionName,
InboundUsageEvent,
Metric,
MetricName,
OutboundUsageEvent,
Outcome,
)
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.proxy.spend_tracking.usage_meter import (
DurableUsageMeter,
UsageCaptureError,
UsageCaptureSettings,
usage_delivery_targets,
)
from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy._types import SpendLogsPayload
from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite
class UsageAcknowledgementError(UsageCaptureError):
def __init__(self, cause: Exception) -> None:
super().__init__("Usage persistence did not acknowledge commit")
self.__cause__ = cause
@dataclass(frozen=True, slots=True)
class CapturedUsage:
payload: SpendLogsPayload
writes: tuple[UsageWrite, ...]
receipt: asyncio.Future[None]
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_FIELDS: Final = TypeAdapter(dict[str, object])
_INSTANT: Final = TypeAdapter(datetime)
_DIMENSIONS: Final = (
("api_key", DimensionName.KEY),
("user", DimensionName.USER),
("team_id", DimensionName.TEAM),
("organization_id", DimensionName.ORGANIZATION),
("end_user", DimensionName.END_USER),
("model", DimensionName.MODEL),
("model_group", DimensionName.MODEL_GROUP),
("model_id", DimensionName.DEPLOYMENT),
("custom_llm_provider", DimensionName.PROVIDER),
("mcp_namespaced_tool_name", DimensionName.TOOL),
("agent_id", DimensionName.AGENT),
)
_USAGE_METRICS: Final = tuple(
metric
for metric in MetricName
if metric not in (MetricName.SPEND, MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS)
)
def _fields(value: object) -> Mapping[str, object]:
return _FIELDS.validate_python(value) if isinstance(value, Mapping) else {}
def _text(value: object) -> str | None:
return value if isinstance(value, str) and value else None
def _utc_time(value: datetime | str) -> datetime:
parsed: Final = _INSTANT.validate_python(value)
return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed.astimezone(timezone.utc)
def _json(value: object) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
def _usage_metrics(usage: Mapping[str, JsonValue] | None) -> tuple[Metric, ...] | None:
if usage is None:
return None
return tuple(
Metric(name=metric, value=Decimal(str(value)))
for metric in _USAGE_METRICS
if isinstance(value := usage.get(metric.value), (int, float)) and not isinstance(value, bool)
)
@dataclass(frozen=True, slots=True)
class UsageCaptureContext:
caller_id: str | None
attempt_id: str
parent_request_id: str | None
retry_of_attempt_id: str | None
job_id: str | None
charge_eligible: bool
request_count_eligible: bool
ownership: str
pricing: str
finalized_cost: str | None = None
allocation: str | None = None
outcome: Outcome | None = None
started_at: datetime | None = None
ended_at: datetime | None = None
phase: str = "execution"
extra_dimensions: tuple[Dimension, ...] = ()
provider_timestamps: str = "{}"
usage_source: str = "measured"
included_attempt_ids: tuple[str, ...] = ()
related_attempt_ids: tuple[str, ...] = ()
def _database_usage_write(event: InboundUsageEvent | OutboundUsageEvent, envelope: Mapping[str, object]) -> UsageWrite:
from sys import maxsize
from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure
normalized: Final = _FIELDS.validate_python(safe_json_structure(envelope, max_depth=maxsize))
if normalized.get("event") != event.model_dump(mode="json"):
raise UsageCaptureError("Usage event text cannot be normalized without changing identity or attribution")
serialized: Final = _json(normalized)
_JSON_OBJECT.validate_json(serialized)
return UsageWrite(event, serialized, usage_delivery_targets(event.dimensions))
def usage_writes(
payload: SpendLogsPayload, usage_json: str | None, context: UsageCaptureContext
) -> tuple[UsageWrite, ...]:
fields: Final = _FIELDS.validate_python(payload)
tags: Final = TypeAdapter(tuple[str, ...]).validate_json(payload["request_tags"])
dimensions: Final = (
*(Dimension(name=name, value=value) for field, name in _DIMENSIONS if (value := _text(fields.get(field)))),
*(Dimension(name=DimensionName.TAG, value=tag) for tag in dict.fromkeys(tags)),
*context.extra_dimensions,
)
outcome: Final = context.outcome or (Outcome.FAILED if payload["status"] == "failure" else Outcome.SUCCEEDED)
started: Final = context.started_at or _utc_time(payload["startTime"])
ended: Final = context.ended_at or _utc_time(payload["endTime"])
usage: Final = None if usage_json is None else _JSON_OBJECT.validate_json(usage_json)
outbound: Final = OutboundUsageEvent(
event_id=context.attempt_id,
attempt_id=context.attempt_id,
request_id=context.caller_id,
retry_of_attempt_id=context.retry_of_attempt_id,
call_type=CallTypes(payload["call_type"]),
started_at=started,
ended_at=ended,
outcome=outcome,
dimensions=dimensions,
metrics=_usage_metrics(None if context.usage_source == "unknown" else usage),
)
envelope: Final = {
"event": outbound.model_dump(mode="json"),
"usage": usage,
"usage_source": "unknown" if usage is None else context.usage_source,
"included_attempt_ids": context.included_attempt_ids,
"related_attempt_ids": context.related_attempt_ids,
"pricing": _JSON_OBJECT.validate_json(context.pricing),
"charge_eligible": context.charge_eligible,
"request_count_eligible": context.request_count_eligible,
"ownership": context.ownership,
"parent_request_id": context.parent_request_id,
"job_id": context.job_id,
"phase": context.phase,
"provider_timestamps": _JSON_OBJECT.validate_json(context.provider_timestamps),
"finalized_cost": None
if context.finalized_cost is None
else _JSON_OBJECT.validate_json(context.finalized_cost),
"allocation": None if context.allocation is None else _JSON_OBJECT.validate_json(context.allocation),
}
outbound_write: Final = _database_usage_write(outbound, envelope)
if not context.request_count_eligible or context.caller_id is None:
return (outbound_write,)
inbound: Final = InboundUsageEvent(
event_id=f"{context.caller_id}:inbound",
request_id=context.caller_id,
call_type=outbound.call_type,
started_at=started,
ended_at=ended,
outcome=outcome,
dimensions=dimensions,
metrics=(
Metric(name=MetricName.REQUESTS, value=Decimal(1)),
Metric(name=MetricName.SUCCESSFUL_REQUESTS, value=Decimal(int(outcome is Outcome.SUCCEEDED))),
Metric(name=MetricName.FAILED_REQUESTS, value=Decimal(int(outcome is not Outcome.SUCCEEDED))),
),
)
return (
_database_usage_write(inbound, {"event": inbound.model_dump(mode="json")}),
outbound_write,
)
_INCLUDED_IN_PARENT: Final = object()
_PRICING_FIELDS: Final = frozenset(
{
"service_tier",
"data_residency",
"vertex_location",
"region_name",
"base_model",
"custom_pricing",
"input_cost_per_token",
"output_cost_per_token",
"input_cost_per_second",
"output_cost_per_second",
"input_cost_per_character",
"output_cost_per_character",
"size",
"quality",
"n",
"duration",
"seconds",
"search_context_size",
"additional_response_cost",
}
)
def _selected_pricing(fields: Mapping[str, object]) -> Mapping[str, JsonValue]:
return _JSON_OBJECT.validate_python(
{
key: value
for key, value in fields.items()
if key in _PRICING_FIELDS or key in CustomPricingLiteLLMParams.model_fields
}
)
def _response_fields(response: object) -> Mapping[str, object]:
if isinstance(response, BaseModel):
return _FIELDS.validate_python(response.model_dump())
return _fields(response)
def _job_time(response: Mapping[str, object], key: str) -> datetime | None:
stamp: Final = response.get(key)
if isinstance(stamp, (int, float)) and not isinstance(stamp, bool):
return datetime.fromtimestamp(stamp, tz=timezone.utc)
return None
def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], response: object) -> UsageCaptureContext:
from dataclasses import asdict
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, is_unbilled_non_inference_call
from litellm.litellm_core_utils.litellm_logging import (
Logging,
UsageCaptureObservation,
UsageObservation,
mask_api_base_credentials,
)
from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost
from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
from litellm.proxy.spend_tracking.spend_tracking_utils import get_request_model_access_groups
metadata: Final = _JSON_OBJECT.validate_json(payload["metadata"])
params: Final = _fields(kwargs.get("litellm_params"))
request_metadata: Final = _fields(params.get("litellm_metadata") or params.get("metadata"))
standard: Final = _fields(kwargs.get("standard_logging_object"))
logging_obj: Final = kwargs.get("litellm_logging_obj")
raw_response: Final = _response_fields(response)
hidden: Final = _fields(get_hidden_params(response))
tool: Final = _fields(kwargs.get("mcp_tool_call_metadata"))
tool_pricing: Final = _fields(tool.get("mcp_server_cost_info"))
caller: Final = payload["litellm_call_id"]
cached: Final = payload["cache_hit"].lower() == "true"
included_in_parent: Final = kwargs.get("_accounting_internal_aggregate") is _INCLUDED_IN_PARENT
billed_id: Final = payload["request_id"]
terminal: Final = raw_response.get("status") in {
"completed",
"complete",
"failed",
"cancelled",
"expired",
"incomplete",
}
deferred: Final = billed_id.endswith(("_batch_cost", "_background_cost")) and terminal
management: Final = (
(billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal)
or (is_background_response(response) and not terminal)
or is_unbilled_non_inference_call(payload["call_type"], request_metadata, response)
)
attempt: Final = (
f"{caller}_cache_hit"
if cached and caller
else f"{caller}:status"
if management
else f"{billed_id}:included"
if included_in_parent
else billed_id
)
if not attempt or attempt == "None" or (cached and not caller):
raise UsageCaptureError("Usage capture requires a stable execution identity")
retry: Final = metadata.get(BILLED_RETRY_ATTEMPT_METADATA_KEY) is True
origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)
parent: Final = _text(kwargs.get("parent_request_id")) or _text(request_metadata.get("parent_request_id"))
finalized_value: Final = (
logging_obj.finalized_cost if isinstance(logging_obj, Logging) else kwargs.get("_accounting_finalized_cost")
)
finalized: Final = finalized_value if isinstance(finalized_value, FinalizedCost) else None
allocation_value: Final = (
logging_obj.cost_allowance_allocation
if isinstance(logging_obj, Logging)
else kwargs.get("_accounting_allocation")
)
allocation: Final = allocation_value if isinstance(allocation_value, SearchAllowanceAllocation) else None
pricing_snapshot: Final = kwargs.get("_accounting_pricing_snapshot")
observation: Final = kwargs.get("_accounting_usage_snapshot")
captured_pricing: Final = {
"model_map_information": (
_JSON_OBJECT.validate_json(pricing_snapshot.model_map_information)
if isinstance(pricing_snapshot, PricingCapture)
else metadata.get("model_map_information")
),
"cost_breakdown": metadata.get("cost_breakdown"),
**(
_JSON_OBJECT.validate_json(pricing_snapshot.inputs)
if isinstance(pricing_snapshot, PricingCapture)
else _pricing_inputs(kwargs, logging_obj, response)
),
"tool": {
"name": tool.get("name"),
"cost_per_query": _fields(tool_pricing.get("tool_name_to_cost_per_query")).get(
_text(tool.get("name")) or "", tool_pricing.get("default_cost_per_query")
),
}
if tool
else None,
"response_inputs": {
"audio_transcription_duration": hidden.get("audio_transcription_duration", raw_response.get("duration")),
"billed_units": _fields(raw_response.get("meta")).get("billed_units"),
"batch_models": hidden.get("batch_models"),
"batch_successful_requests": hidden.get("batch_successful_requests"),
"batch_failed_requests": hidden.get("batch_failed_requests"),
"service_tier": raw_response.get("service_tier"),
"region_name": hidden.get("region_name"),
},
"reported_cost": observation.response_cost
if isinstance(observation, UsageCaptureObservation)
else standard.get(
"response_cost", kwargs.get("response_cost", _fields(get_hidden_params(response)).get("response_cost"))
),
"adjustments": (
_JSON_OBJECT.validate_json(pricing_snapshot.adjustments)
if isinstance(pricing_snapshot, PricingCapture)
else None
),
}
return UsageCaptureContext(
caller_id=None if deferred else parent if retry or included_in_parent else caller,
attempt_id=attempt,
parent_request_id=parent,
retry_of_attempt_id=_text(request_metadata.get("retry_of_attempt_id")),
job_id=_text(raw_response.get("id")) if deferred or management else None,
charge_eligible=not cached and not included_in_parent and not management,
request_count_eligible=bool(caller) and not origin and not retry and not deferred and not included_in_parent,
ownership="included_in_parent"
if included_in_parent
else "aggregate"
if kwargs.get("additional_response_cost") is not None
else "attempt",
pricing=_json(captured_pricing),
finalized_cost=_json(
{
"response_cost": finalized.amount,
"cost_breakdown": finalized.logging_breakdown(),
"allowance_discount": finalized.allowance_discount,
}
)
if finalized is not None
else (
_json({"response_cost": standard.get("response_cost"), "cost_breakdown": standard.get("cost_breakdown")})
if standard.get("cost_finalized") is True
else None
),
allocation=_json(asdict(allocation)) if allocation is not None else None,
started_at=_job_time(raw_response, "created_at") if deferred else None,
ended_at=(
_job_time(raw_response, "completed_at")
or _job_time(raw_response, "failed_at")
or _job_time(raw_response, "cancelled_at")
or _job_time(raw_response, "expired_at")
)
if deferred
else None,
outcome=(
Outcome.CANCELLED
if raw_response.get("status") == "cancelled"
or isinstance(response, asyncio.CancelledError)
or getattr(response, "status_code", None) == 499
else Outcome.FAILED
if raw_response.get("status") in {"failed", "expired", "incomplete"} or isinstance(response, BaseException)
else None
),
phase="terminal" if deferred else "management" if management else "execution",
usage_source="estimated"
if kwargs.get("_accounting_usage_observed") is UsageObservation.ESTIMATED
else "measured",
included_attempt_ids=TypeAdapter(tuple[str, ...]).validate_python(
kwargs.get("_accounting_included_attempt_ids", ())
),
related_attempt_ids=tuple(write.event.event_id for write in _prior_writes(kwargs)),
provider_timestamps=_json(
{
key: raw_response.get(key)
for key in ("created_at", "completed_at", "failed_at", "cancelled_at", "expired_at")
if raw_response.get(key) is not None
}
)
if deferred
else "{}",
extra_dimensions=(
*(
(Dimension(name=DimensionName.TOOL, value=tool_name),)
if not payload.get("mcp_namespaced_tool_name")
and (tool_name := _text(tool.get("namespaced_tool_name")))
else ()
),
*(
(Dimension(name=DimensionName.API_BASE, value=mask_api_base_credentials(api_base)),)
if (api_base := _text(payload["api_base"]))
else ()
),
*(
Dimension(name=DimensionName.MODEL_ACCESS_GROUP, value=group)
for group in get_request_model_access_groups(kwargs)
),
*(
(Dimension(name=DimensionName.PROJECT, value=project),)
if (project := _text(request_metadata.get("user_api_key_project_id")))
else ()
),
*(
(Dimension(name=DimensionName.SERVICE_TIER, value=tier),)
if (tier := _text(raw_response.get("service_tier") or kwargs.get("service_tier")))
else ()
),
),
)
class _CaptureBuilder:
def __init__(self, kwargs: Mapping[str, object], response: object) -> None:
self.kwargs: Final = kwargs
self.response: Final = response
self.writes: tuple[UsageWrite, ...] = ()
def observe(self, payload: SpendLogsPayload, usage: str | None) -> None:
from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation, UsageObservation
snapshot: Final = self.kwargs.get("_accounting_usage_snapshot")
if isinstance(snapshot, UsageCaptureError):
raise snapshot
effective: Final = snapshot.usage_json if isinstance(snapshot, UsageCaptureObservation) else usage
parsed: Final = _JSON_OBJECT.validate_json(effective) if effective is not None else {}
batch: Final = _fields(parsed.get("batch_usage_by_model"))
unknown: Final = (bool(batch) or bool(parsed.get("batch_unknown_usage_requests"))) and all(
value is None for value in batch.values()
)
observed: Final = (
_json(
{
"batch_usage_by_model": batch,
"batch_unknown_usage_requests": parsed.get("batch_unknown_usage_requests"),
}
)
if unknown
else effective
)
context: Final = capture_context(payload, self.kwargs, self.response)
provenance: Final = (
"unknown"
if unknown
else "estimated"
if isinstance(snapshot, UsageCaptureObservation) and snapshot.provenance is UsageObservation.ESTIMATED
else context.usage_source
)
self.writes = usage_writes(payload, observed, replace(context, usage_source=provenance))
def _prior_captures(kwargs: Mapping[str, object]) -> tuple[CapturedUsage, ...]:
prior: Final = kwargs.get("_accounting_prior_captures", ())
return (
tuple(
item for item in TypeAdapter(tuple[object, ...]).validate_python(prior) if isinstance(item, CapturedUsage)
)
if isinstance(prior, tuple)
else ()
)
def _prior_writes(kwargs: Mapping[str, object]) -> tuple[UsageWrite, ...]:
return tuple(chain.from_iterable(item.writes for item in _prior_captures(kwargs)))
def capture_callback(
meter: DurableUsageMeter,
kwargs: Mapping[str, object],
response: object,
start_time: datetime,
end_time: datetime,
) -> CapturedUsage:
from litellm.proxy.db.db_spend_update_writer import get_llm_router
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
existing: Final = kwargs.get("_accounting_capture")
if isinstance(existing, CapturedUsage):
return existing
builder: Final = _CaptureBuilder(kwargs, response)
payload: Final = get_logging_payload(
dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe
)
prerequisites: Final = tuple(item.receipt for item in _prior_captures(kwargs))
return CapturedUsage(payload, builder.writes, meter.submit(builder.writes, prerequisites=prerequisites))
def captured_callback_kwargs(
kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime
) -> Mapping[str, object]:
from litellm.proxy.proxy_server import usage_capture
usage_meter: Final = usage_capture.meter
if usage_meter is None:
return kwargs
try:
return {**kwargs, "_accounting_capture": capture_callback(usage_meter, kwargs, response, start_time, end_time)}
except Exception: # noqa: BLE001 # capture failure must not suppress legacy billing
verbose_proxy_logger.exception("Usage capture failed before legacy cost tracking")
return kwargs
async def capture_proxy_response(logging_obj: object, response: object, *, background: bool = False) -> None:
from fastapi import HTTPException
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import general_settings, usage_capture
settings: Final = _FIELDS.validate_python(general_settings)
usage_meter: Final = usage_capture.meter
if usage_meter is None or not isinstance(logging_obj, Logging):
return
sync_ack: Final = (
settings.get("accounting_sync_ack") is True
and logging_obj.stream is not True
and not background
and logging_obj.call_type in {"acompletion", "atext_completion", "aembedding", "aresponses"}
)
try:
capture_logging_usage(usage_meter, logging_obj, response, logging_obj.start_time, datetime.now(timezone.utc))
captured: Final = logging_obj.model_call_details.get("_accounting_capture")
if sync_ack:
if not isinstance(captured, CapturedUsage):
raise UsageCaptureError("Usage capture was not admitted")
timeout: Final = TypeAdapter(float).validate_python(
settings.get("accounting_sync_ack_timeout_seconds", 20.0)
)
await asyncio.wait_for(asyncio.shield(captured.receipt), timeout=timeout)
except Exception as error: # noqa: BLE001 # the configured response policy owns every capture failure
verbose_proxy_logger.exception("Usage capture failed before the response boundary")
if sync_ack and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed":
acknowledgement: Final = UsageAcknowledgementError(error)
raise HTTPException(503, str(acknowledgement)) from acknowledgement
def capture_logging_usage(
meter: DurableUsageMeter,
logging_obj: object,
response: object,
start_time: datetime,
end_time: datetime,
freeze_only: bool = False,
) -> CapturedUsage | UsageCaptureError | None:
from litellm._internal_context import internal_parent_request_id, is_internal_call
from litellm.batches.batch_utils import batch_cost_is_final
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import LiteLLMBatch
if not isinstance(logging_obj, Logging):
return
internal: Final = is_internal_call.get()
if isinstance(response, LiteLLMBatch) and batch_cost_is_final(response) and response.usage is None:
return
try:
if not internal and isinstance(logging_obj.model_call_details.get("_accounting_capture"), CapturedUsage):
return capture_callback(meter, logging_obj.model_call_details, response, start_time, end_time)
normalized_result: Final = logging_obj.usage_capture_response(response)
prepared: Final = logging_obj.prepared_cost
normalized: Final = (
{**_response_fields(normalized_result), "usage": prepared.usage.model_dump()}
if prepared is not None
else normalized_result
)
if normalized is None:
return
details: Final = _FIELDS.validate_python(logging_obj.model_call_details)
params: Final = _fields(details.get("litellm_params"))
metadata: Final = _fields(params.get("litellm_metadata") or params.get("metadata"))
if metadata.get("batch_ignore_default_logging") is True:
return
existing_pricing: Final = details.get("_accounting_pricing_snapshot")
pricing: Final = (
existing_pricing
if isinstance(existing_pricing, PricingCapture) and not internal
else _capture_pricing(logging_obj, normalized, details)
)
if freeze_only:
if not internal:
logging_obj.record_usage_pricing(pricing)
return
captured: Final = capture_callback(
meter,
{
**details,
"litellm_logging_obj": logging_obj,
"_accounting_capture": None if internal else details.get("_accounting_capture"),
"_accounting_pricing_snapshot": pricing,
"_accounting_internal_aggregate": _INCLUDED_IN_PARENT if internal else None,
"parent_request_id": internal_parent_request_id.get() if internal else details.get("parent_request_id"),
},
normalized,
start_time,
end_time,
)
if not internal:
logging_obj.record_usage_capture(captured)
except Exception as error: # noqa: BLE001 # failed capture must not prevent normal logging and legacy charging
verbose_proxy_logger.exception("Usage capture failed at the execution boundary")
return UsageCaptureError(str(error))
else:
return captured
@dataclass(frozen=True, slots=True)
class PricingCapture:
model_map_information: str
adjustments: str
inputs: str
def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[str, object]) -> PricingCapture:
import litellm
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
params: Final = _fields(details.get("litellm_params"))
prepared: Final = logging_obj.prepared_cost
provider: Final = _text(details.get("custom_llm_provider"))
builtins: Final = logging_obj.standard_built_in_tools_params
pricing: Final = (
prepared.payload["model_map_information"]
if prepared is not None
else StandardLoggingPayloadSetup.get_model_cost_information(
base_model=logging_obj.get_deployment_model_for_cost(),
custom_pricing=bool(params.get("custom_pricing")),
custom_llm_provider=_text(details.get("custom_llm_provider")),
init_response_obj=normalized,
api_base=_text(params.get("api_base")),
router_model_id=logging_obj.get_router_model_id(),
)
)
adjustments: Final = _json(
{
"discount": resolve_cost_discount(
litellm.cost_discount_config,
provider,
logging_obj.get_deployment_model_for_cost(),
_text(params.get("region_name") or params.get("vertex_location")),
),
"margin": litellm.cost_margin_config.get(provider or "", litellm.cost_margin_config.get("global")),
"web_search_options": (
_selected_pricing(_fields(builtins.get("web_search_options")))
if builtins.get("web_search_options") is not None
else None
),
"file_search": builtins.get("file_search") is not None,
}
)
batch_usage: Final = _fields(_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model"))
return PricingCapture(
_json(pricing),
adjustments,
_json(
{
**_pricing_inputs(details, logging_obj, normalized),
"batch_models": {
model: StandardLoggingPayloadSetup.get_model_cost_information(
base_model=model,
custom_pricing=bool(params.get("custom_pricing")),
custom_llm_provider=provider,
init_response_obj={"model": model},
api_base=_text(params.get("api_base")),
router_model_id=logging_obj.get_router_model_id(),
)
for model in batch_usage
},
}
),
)
def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]:
from litellm.litellm_core_utils.litellm_logging import Logging
return {
"request": _selected_pricing(kwargs),
"deployment": _selected_pricing(_fields(kwargs.get("litellm_params"))),
"optional_params": _selected_pricing(_fields(kwargs.get("optional_params"))),
"logging": logging_obj.usage_capture_pricing(response) if isinstance(logging_obj, Logging) else None,
"model_info": _selected_pricing(
_fields(
_fields(
_fields(kwargs.get("litellm_params")).get("litellm_metadata")
or _fields(kwargs.get("litellm_params")).get("metadata")
).get("model_info")
)
),
}
def _terminal_facts(write: UsageWrite) -> Mapping[str, JsonValue] | None:
payload: Final = _JSON_OBJECT.validate_json(write.envelope)
if payload.get("phase") != "terminal":
return None
event: Final = _JSON_OBJECT.validate_python(payload["event"])
return {
**{
key: value
for key, value in payload.items()
if key not in {"event", "pricing", "finalized_cost", "allocation"}
},
"event": {key: value for key, value in event.items() if key not in {"started_at", "ended_at"}},
}
def _reuse_terminal_snapshot(candidate: UsageWrite, accepted: UsageWrite) -> UsageWrite:
facts: Final = _terminal_facts(candidate)
if facts is None or facts != _terminal_facts(accepted) or set(candidate.targets) != set(accepted.targets):
raise AccountingConflictError("Repeated terminal job disagrees with its accepted usage or attribution")
return accepted
class UsageBatchWriter:
def __init__(
self,
insert: Callable[[tuple[UsageWrite, ...]], Awaitable[None]],
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]],
) -> None:
self._insert: Final = insert
self._load: Final = load
async def _restore(self, writes: tuple[UsageWrite, ...]) -> tuple[UsageWrite, ...]:
terminal: Final = tuple(write for write in writes if _terminal_facts(write) is not None)
if not terminal:
return writes
identities: Final = tuple(dict.fromkeys(write.event.event_id for write in terminal))
stored: Final = await self._load(identities)
# Re-read the first accepted observation and pricing before the ledger's unchanged id/hash check.
snapshots: Final = {write.event.event_id: write for write in (*reversed(terminal), *stored)}
return tuple(
_reuse_terminal_snapshot(write, snapshots[write.event.event_id])
if write.event.event_id in snapshots
else write
for write in writes
)
async def write(self, writes: tuple[UsageWrite, ...]) -> None:
restored: Final = await self._restore(writes)
try:
await self._insert(restored)
except AccountingConflictError:
if not any(_terminal_facts(write) is not None for write in writes):
raise
await self._insert(await self._restore(writes))
class UsageCaptureRuntime:
def __init__(self) -> None:
self.meter: DurableUsageMeter | None = None
def start(
self, write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]], settings: UsageCaptureSettings
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
if self.meter is not None:
if self.meter.settings != settings:
raise ValueError("Changing accounting_capture requires a worker restart")
return
meter: Final = DurableUsageMeter(write_batch, settings)
self.meter = meter
Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage(
meter, logger, response, start, end, freeze
)
async def stop(self, timeout: float) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
if self.meter is None:
return
try:
await self.meter.stop(timeout)
finally:
Logging.usage_capture_callback = None
self.meter = None

View file

@ -163,7 +163,12 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.deployment_pricing import pricing_models
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
from litellm.litellm_core_utils.litellm_logging import (
BilledRoundsDispatch,
Logging,
UsageObservation,
deployment_pricing_model_info,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -1739,6 +1744,39 @@ def _failure_fields_to_lift(
if _prompt_estimate is not None
else ()
)
from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation
_recorded_observation: Final[object] = _model_call_details.get("_accounting_usage_observed")
_current_usage: Final = (
_current_failure_usage_to_lift(
_model_call_details,
request_data,
_model_call_details.get("current_attempt_api_call_start_time", _first_handoff) is not None,
_billed,
_deployment_id,
_base_model,
dispatched_call_type,
)
if Logging.usage_capture_callback is not None
and (_billed or _model_call_details.get("combined_usage_object") is not None)
else None
)
_current_observation: Final = (
_recorded_observation
if isinstance(_recorded_observation, UsageObservation)
else UsageObservation.UNKNOWN
if _current_usage is None
else UsageObservation.KNOWN
if _model_call_details.get("combined_usage_object") is not None
else UsageObservation.ESTIMATED
)
_capture_observation: Final = UsageCaptureObservation.from_usage(
_current_usage[0] if _current_usage is not None and isinstance(_current_usage[0], Usage) else None,
_current_observation,
float(_current_usage[1])
if _current_usage is not None and isinstance(_current_usage[1], (int, float))
else None,
)
_entries: Final = (
("start_time", _model_call_details.get("start_time")),
("first_api_call_start_time", _first_handoff),
@ -1747,6 +1785,16 @@ def _failure_fields_to_lift(
("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)),
("standard_logging_object", _model_call_details.get("standard_logging_object")),
("billed_failed_attempts", _earlier_attempts or None),
("_accounting_capture", _model_call_details.get("_accounting_capture")),
("_accounting_pricing_snapshot", _model_call_details.get("_accounting_pricing_snapshot")),
("_accounting_finalized_cost", _logging_obj.finalized_cost if isinstance(_logging_obj, Logging) else None),
(
"_accounting_allocation",
_logging_obj.cost_allowance_allocation if isinstance(_logging_obj, Logging) else None,
),
("_accounting_usage_observed", _current_observation if Logging.usage_capture_callback is not None else None),
("_accounting_usage_snapshot", _capture_observation if Logging.usage_capture_callback is not None else None),
("_accounting_prior_captures", _model_call_details.get("_accounting_prior_captures")),
)
return MappingProxyType({key: value for key, value in _entries if value is not None})

View file

@ -223,6 +223,10 @@ async def _execute_query_pipeline(
"""
Execute the RAG query pipeline.
"""
parent_request_id: Final[str | None] = TypeAdapter[str | None](str | None).validate_python(
TypeAdapter(dict[str, object]).validate_python(kwargs).get("litellm_call_id")
)
# Extract router from kwargs - use it for completion if available
# to properly resolve virtual model names
router: Final[Router | None] = kwargs.pop("router", None)
@ -252,7 +256,7 @@ async def _execute_query_pipeline(
forwarded_search_params: Final = MappingProxyType(
{**provider_search_params, **kwargs, **filter_search_params, **store_search_params}
)
with internal_sub_call():
with internal_sub_call(parent_request_id):
search_response: Final = await litellm.vector_stores.asearch(
vector_store_id=retrieval_config["vector_store_id"],
query=query_text,
@ -286,7 +290,7 @@ async def _execute_query_pipeline(
if rerank and rerank.get("enabled"):
documents: Final = RAGQuery.extract_documents_from_search(search_response)
if documents:
with internal_sub_call():
with internal_sub_call(parent_request_id):
rerank_response = await litellm.arerank(
model=rerank["model"],
query=query_text,
@ -304,7 +308,7 @@ async def _execute_query_pipeline(
modified_messages: Final = messages[:-1] + [context_message] + [messages[-1]]
# Use router if available to properly resolve virtual model names
with internal_sub_call():
with internal_sub_call(parent_request_id):
if router is not None:
response = await router.acompletion(
model=model,

View file

@ -437,7 +437,7 @@ async def aresponses_api_with_mcp(
#########################################################
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.carry_failed_attempt_usage()
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
response: Final = await aresponses(
input=input,
model=model,
@ -514,7 +514,9 @@ async def aresponses_api_with_mcp(
tool_calls=tool_calls, tool_results=tool_results
)
with internal_sub_call():
with internal_sub_call(
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
):
final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call(
follow_up_input=follow_up_input,
model=model,

View file

@ -663,7 +663,7 @@ async def acompletion_with_mcp(
logging_obj: Final = kwargs.get("litellm_logging_obj")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.carry_failed_attempt_usage()
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
initial_response: Final = await litellm_acompletion(**initial_call_args)
first_round_breakdown: Final = logging_obj.cost_breakdown if isinstance(logging_obj, LiteLLMLoggingObj) else None
@ -713,7 +713,7 @@ async def acompletion_with_mcp(
follow_up_call_args["messages"] = follow_up_messages
follow_up_call_args["stream"] = stream
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
final_response: Final = await litellm_acompletion(**follow_up_call_args)
if not isinstance(final_response, ModelResponse):

View file

@ -3735,6 +3735,11 @@ class Router(DeprecatedRouterMembers):
priced_partial._hidden_params = { # pyright: ignore[reportPrivateUsage] # no public accessor
key: value for key, value in get_hidden_params_dict(partial_response).items() if key != "response_cost"
}
if LiteLLMLogging.usage_capture_callback is not None:
from litellm._internal_context import internal_sub_call
with internal_sub_call(cut_logging.litellm_call_id):
cut_logging.capture_usage_before_logging(priced_partial)
return ContinuedPartial(
usage=BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[partial_usage, inherited_usage]

View file

@ -676,6 +676,12 @@ class CallTypes(str, Enum):
retrieve_fine_tuning_job = "retrieve_fine_tuning_job"
responses = "responses"
aresponses = "aresponses"
get_responses = "get_responses"
aget_responses = "aget_responses"
delete_responses = "delete_responses"
adelete_responses = "adelete_responses"
cancel_responses = "cancel_responses"
acancel_responses = "acancel_responses"
alist_input_items = "alist_input_items"
llm_passthrough_route = "llm_passthrough_route"
allm_passthrough_route = "allm_passthrough_route"

View file

@ -1595,6 +1595,7 @@ def dispatch_success_logging(
is_litellm_internal_call: bool,
) -> None:
if is_litellm_internal_call:
logging_obj.capture_usage_before_logging(result, start_time, end_time)
return
_schedule_async_success_logging(
@ -1633,9 +1634,14 @@ def _schedule_async_success_logging(
if not logging_obj.claim_async_success_log():
return
if logging_obj.defer_async_logging and not is_completion_with_fallbacks:
logging_obj.capture_usage_before_logging(result, start_time, end_time, freeze_only=True)
def _enqueue_async_logging() -> None:
from litellm.responses.streaming_iterator import spawn_logging_task
if not is_completion_with_fallbacks:
logging_obj.capture_usage_before_logging(result, start_time, end_time)
if logging_obj.defer_async_logging:
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=result, start_time=start_time, end_time=end_time
@ -2203,6 +2209,7 @@ def client(original_function):
# LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated
verbose_logger.info("Wrapper: Completed Call, calling success_handler")
logging_obj.capture_usage_before_logging(TypeAdapter(object).validate_python(result), start_time, end_time)
if not is_internal_call.get():
# Copy the current context to propagate it to the background thread
# This is essential for OpenTelemetry span context propagation

View file

@ -3436,3 +3436,50 @@ def test_invalid_batch_envelopes_preserve_recoverable_cost(
assert result.usage.prompt_tokens == (200 if include_valid else 0)
assert result.usage.completion_tokens == (40 if include_valid else 0)
assert (result.successful_requests, result.failed_requests) == (2 if include_valid else 0, 1)
def test_unpriced_batch_retains_reported_usage_by_model_and_marks_missing_usage():
rows: Final = [
_success_row(model="unregistered-batch-model-a", usage=_usage(4, 2)),
_success_row(model="unregistered-batch-model-a", usage=_usage(3, 1)),
_success_row(model="unregistered-batch-model-b", usage=_usage(5, 2)),
{"response": {"status_code": 200, "body": {"model": "unregistered-batch-model-c"}}},
]
result: Final = bu.batch_cost_and_usage(rows, "openai")
captured: Final = result.usage.model_dump()
assert (result.usage.prompt_tokens, result.usage.completion_tokens) == (12, 5)
assert result.cost == 0
by_model: Final = captured["batch_usage_by_model"]
assert by_model["unregistered-batch-model-a"]["total_tokens"] == 10
assert by_model["unregistered-batch-model-b"]["total_tokens"] == 7
assert by_model["unregistered-batch-model-c"] is None
assert captured["batch_unknown_usage_requests"] == 1
def test_batch_ocr_preserves_additive_page_quantities_without_changing_cost():
rows: Final = [
{
"response": {
"status_code": 200,
"body": {
"model": "ocr-quantity-control",
"usage_info": {"pages_processed": pages, "doc_size_bytes": size},
},
}
}
for pages, size in ((2, 100), (3, 200))
]
result: Final = bu.batch_cost_and_usage(rows, "mistral", model_info={"ocr_cost_per_page_batches": 0.01})
captured: Final = result.usage.model_dump()["batch_usage_by_model"]["ocr-quantity-control"]
assert captured["pages_processed"] == 5
assert captured["doc_size_bytes"] == 300
assert result.cost == pytest.approx(5 * 0.01)
def test_native_vertex_capture_retains_unknown_models_without_changing_legacy_counts():
rows: Final = [{"request": {}, "response": {"modelVersion": "missing-usage-model", "candidates": []}}]
result: Final = bu.calculate_vertex_ai_batch_cost_and_usage(rows)
captured: Final = result.usage.model_dump()
assert captured["batch_usage_by_model"] == {"missing-usage-model": None}
assert captured["batch_unknown_usage_requests"] == 1
assert (result.cost, result.successful_requests, result.failed_requests) == (0, 0, 1)

View file

@ -467,6 +467,9 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
"add_message",
"arun_thread_stream",
"aresponses",
"aget_responses",
"adelete_responses",
"acancel_responses",
"aresponses_websocket",
"alist_input_items",
"acreate_fine_tuning_job",

View file

@ -3949,3 +3949,41 @@ async def test_async_post_call_failure_hook_disable_pre_routing_error_logs(
user_api_key_dict=user_api_key_dict,
)
assert len(writer.rows) == rows_written
@pytest.mark.asyncio
async def test_failure_capture_overload_does_not_skip_budget_release(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier, _write
database: Final = CommitBarrier()
runtime: Final = UsageCaptureRuntime()
runtime.start(database.write, UsageCaptureSettings(batch_size=2, max_pending_records=2))
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "prisma_client", None)
meter: Final = runtime.meter
assert meter is not None
receipt: Final = meter.submit((_write("occupied-one"), _write("occupied-two")))
auth: Final = UserAPIKeyAuth(budget_reservation={"reserved_cost": 0.5, "entries": [], "finalized": False})
request: Final = {
"call_type": "acompletion",
"litellm_call_id": "failed",
"model": "capture-model",
"start_time": _CALL_START,
"litellm_params": {},
}
try:
await database.entered.wait()
await ProxyDBLogger().async_post_call_failure_hook(request, RuntimeError("provider failed"), auth)
assert auth.budget_reservation is not None and auth.budget_reservation["finalized"] is True
assert "Usage capture failed on the request failure path" in caplog.text
assert "_accounting_capture" not in request
finally:
database.release.set()
await runtime.stop(1)
await receipt
assert database.committed == ("occupied-one", "occupied-two")

View file

@ -91,12 +91,20 @@ def response(cost: float = 0.5) -> ModelResponse:
async def test_header_and_logging_share_one_allowance_allocation(
logging_obj: Logging, used: int, expected: float
) -> None:
import json
from litellm.proxy.db.accounting_ledger import UsageWrite
from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage
from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings
counter: Final = Counter(used)
result: Final = response()
costs: Final = await asyncio.gather(*(finalize_response_cost(logging_obj, result, counter, NOW) for _ in range(3)))
assert costs[0] is costs[1] is costs[2]
assert costs[0] is not None
assert costs[0].amount == pytest.approx(expected)
assert logging_obj.cost_allowance_allocation is not None
assert logging_obj.cost_allowance_allocation.discount == pytest.approx(costs[0].allowance_discount)
assert counter.calls == 1
logging_obj._success_handler_helper_fn(result=result, start_time=NOW, end_time=NOW)
payload: Final = logging_obj.model_call_details["standard_logging_object"]
@ -107,6 +115,25 @@ async def test_header_and_logging_share_one_allowance_allocation(
assert payload["cost_finalized"] is True
assert payload["cost_breakdown"] == costs[0].logging_breakdown()
committed: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
for write in writes:
committed.put_nowait(write)
meter: Final = DurableUsageMeter(persist, UsageCaptureSettings())
assert logging_obj.prepared_cost is not None
prepared_usage: Final = logging_obj.prepared_cost.usage.model_dump()
result.usage = Usage(prompt_tokens=999)
capture_logging_usage(meter, logging_obj, result, NOW, NOW)
await meter.close()
committed.get_nowait()
event: Final = json.loads(committed.get_nowait().envelope)
assert event["finalized_cost"]["response_cost"] == pytest.approx(costs[0].amount)
assert event["allocation"]["discount"] == pytest.approx(costs[0].allowance_discount)
assert event["usage"] == prepared_usage
assert counter.calls == 1
@pytest.mark.asyncio
async def test_failed_counter_freezes_full_price_without_a_retry(logging_obj: Logging) -> None:

View file

@ -7543,3 +7543,47 @@ def test_get_logging_payload_stores_end_user_only_while_end_user_cost_tracking_i
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["end_user"] == expected_end_user
@pytest.mark.parametrize("tool_metadata", [{"name": "tool"}, {"name": "tool", "arguments": "invalid"}])
def test_optional_mcp_metadata_does_not_change_legacy_spend_payload(tool_metadata: Mapping[str, object]) -> None:
now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc)
kwargs: Final = {
"call_type": "call_mcp_tool",
"litellm_call_id": "tool-call",
"litellm_params": {},
"litellm_trace_id": "fixed-trace",
"response_cost": 0.5,
}
response: Final = {"id": "tool-call"}
expected: Final = spend_tracking_utils.get_logging_payload(kwargs, response, now, now)
actual: Final = spend_tracking_utils.get_logging_payload(
{**kwargs, "mcp_tool_call_metadata": tool_metadata}, response, now, now
)
assert actual == expected
@pytest.mark.parametrize("capture_enabled", [False, True])
@pytest.mark.parametrize("completion_start", [None, "absent", "reported"])
def test_spend_payload_preserves_unknown_completion_start(capture_enabled: bool, completion_start: str | None) -> None:
start: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc)
end: Final = start + datetime.timedelta(seconds=2)
reported: Final = start + datetime.timedelta(seconds=1)
observations: Final[list[object]] = []
kwargs: Final = {
"call_type": "acompletion",
"litellm_call_id": "timestamp-call",
"litellm_params": {},
**({} if completion_start == "absent" else {"completion_start_time": reported if completion_start else None}),
}
payload: Final = spend_tracking_utils.get_logging_payload(
kwargs,
{"id": "timestamp-response"},
start,
end,
capture_usage=(lambda row, usage: observations.append(row)) if capture_enabled else None,
)
assert payload["completionStartTime"] == (
end if completion_start == "absent" else reported if completion_start else None
)
assert observations == ([payload] if capture_enabled else [])

File diff suppressed because it is too large Load diff

View file

@ -268,8 +268,15 @@ token = MASTER_KEY
@pytest.mark.asyncio
async def test_shutdown_persists_accounting_from_worker_callbacks_before_returning() -> None:
async def test_shutdown_persists_accounting_from_worker_callbacks_before_returning(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from datetime import datetime
from litellm.litellm_core_utils.logging_worker import LoggingWorker
from litellm.proxy.db.accounting_ledger import UsageWrite
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_callback
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.responses.streaming_iterator import spawn_logging_task
worker: Final = LoggingWorker()
@ -279,7 +286,25 @@ async def test_shutdown_persists_accounting_from_worker_callbacks_before_returni
pending: Final[asyncio.Queue[str]] = asyncio.Queue()
persisted: Final[asyncio.Queue[str]] = asyncio.Queue()
captured: Final[asyncio.Queue[str]] = asyncio.Queue()
async def persist_usage(writes: tuple[UsageWrite, ...]) -> None:
for write in writes:
captured.put_nowait(write.event.event_id)
capture: Final = UsageCaptureRuntime()
capture.start(persist_usage, UsageCaptureSettings(flush_interval_seconds=10))
monkeypatch.setattr(litellm.proxy.proxy_server, "usage_capture", capture)
async def enqueue_spend() -> None:
assert capture.meter is not None
capture_callback(
capture.meter,
{"call_type": "acompletion", "litellm_call_id": "shutdown-usage", "litellm_params": {}},
{"id": "shutdown-attempt", "usage": {"total_tokens": 3}},
datetime(2026, 1, 1),
datetime(2026, 1, 1),
)
pending.put_nowait("accounted request")
async def callback() -> None:
@ -305,10 +330,13 @@ async def test_shutdown_persists_accounting_from_worker_callbacks_before_returni
assert waited_for_callback, "Shutdown returned while accounting callbacks were still running"
assert persisted.get_nowait() == "accounted request"
assert pending.empty()
assert (captured.get_nowait(), captured.get_nowait()) == ("shutdown-usage:inbound", "shutdown-attempt")
assert capture.meter is None
finally:
release.set()
await worker.flush()
await worker.stop()
await capture.stop(0)
@pytest.mark.asyncio
@ -4261,3 +4289,255 @@ async def test_shutdown_waits_for_enqueue_producers_before_stopping_worker() ->
assert worker.pending_count == 0
finally:
await worker.stop(discard_pending=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("unsupported", ["database", "collector", "remote", "free_tier"])
async def test_capture_enablement_rejects_unrecoverable_writer_modes(
unsupported: str, monkeypatch: pytest.MonkeyPatch
) -> None:
import litellm
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
monkeypatch.setattr(proxy_server, "usage_capture", UsageCaptureRuntime())
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_capture": {}})
monkeypatch.setattr(proxy_server, "prisma_client", None if unsupported == "database" else object())
monkeypatch.setattr(litellm, "apply_search_free_tier", unsupported == "free_tier")
monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "true" if unsupported == "collector" else "false")
monkeypatch.delenv("SPEND_LOGS_URL", raising=False)
if unsupported == "remote":
monkeypatch.setenv("SPEND_LOGS_URL", "https://writer.invalid")
with pytest.raises(ValueError, match="requires local Prisma"):
proxy_server._configure_usage_capture()
assert proxy_server.usage_capture.meter is None
@pytest.mark.asyncio
async def test_capture_configuration_requires_restart_without_replacing_the_active_writer(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
from litellm.proxy.utils import PrismaClient
runtime: Final = UsageCaptureRuntime()
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True})
with pytest.raises(ValueError, match="requires accounting_capture"):
proxy_server._configure_usage_capture()
client: Final = PrismaClient("postgresql://localhost/unused", ProxyLogging(UserApiKeyCache()))
monkeypatch.setattr(proxy_server, "prisma_client", client)
monkeypatch.setattr(litellm, "apply_search_free_tier", False)
monkeypatch.setenv("LITELLM_COLLECTOR_ENABLED", "false")
monkeypatch.delenv("SPEND_LOGS_URL", raising=False)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_capture": {"batch_size": 4}})
try:
proxy_server._configure_usage_capture()
active: Final = runtime.meter
assert active is not None
assert active.settings.batch_size == 4
proxy_server._configure_usage_capture()
assert runtime.meter is active
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_capture": {"batch_size": 6}})
with pytest.raises(ValueError, match="Changing accounting_capture requires a worker restart"):
proxy_server._configure_usage_capture()
monkeypatch.setattr(proxy_server, "general_settings", {})
with pytest.raises(ValueError, match="Disabling accounting_capture requires a worker restart"):
proxy_server._configure_usage_capture()
assert runtime.meter is active
assert active.settings.batch_size == 4
finally:
await runtime.stop(1)
assert runtime.meter is None
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_ack", [False, True])
@pytest.mark.parametrize(
("path", "payload"),
[
(
"/v1/chat/completions",
{"model": "openai/gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True},
),
("/v1/responses", {"model": "openai/gpt-5-mini", "input": "hi", "background": True}),
(
"/v1/messages",
{"model": "openai/gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 1},
),
],
)
async def test_sync_ack_preserves_async_request_support(
sync_ack: bool,
path: str,
payload: dict[str, object],
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
from fastapi import Request
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.db.accounting_ledger import UsageWrite
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
persisted: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
for write in writes:
persisted.put_nowait(write)
async def authorize(request: Request) -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="test-key", request_route=request.url.path)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setenv("OPENAI_API_KEY", "test-provider-key")
upstream: Final = {
"id": "provider-result",
"object": "chat.completion",
"created": 1,
"model": "gpt-5-mini",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
}
if payload.get("stream"):
chunk: Final = {
**upstream,
"object": "chat.completion.chunk",
"choices": [{"index": 0, "finish_reason": "stop", "delta": {"role": "assistant", "content": "hi"}}],
}
respx_mock.post().respond(
200, text=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n", headers={"content-type": "text/event-stream"}
)
elif path == "/v1/responses":
respx_mock.post().respond(
200,
json={
"id": "resp_background",
"object": "response",
"created_at": 1,
"model": "gpt-5-mini",
"status": "in_progress",
"background": True,
"output": [],
"usage": None,
},
)
else:
respx_mock.post().respond(200, json=upstream)
runtime: Final = UsageCaptureRuntime()
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(
proxy_server, "general_settings", {"accounting_sync_ack": sync_ack, "accounting_sync_ack_timeout_seconds": 0}
)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(
proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": payload["model"],
"litellm_params": {"model": payload["model"], "api_key": "test-provider-key"},
}
],
num_retries=0,
),
)
monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(UserApiKeyCache()))
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger()])
monkeypatch.setitem(proxy_server.app.dependency_overrides, proxy_server.user_api_key_auth, authorize)
runtime.start(persist, UsageCaptureSettings())
try:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=proxy_server.app), base_url="http://test"
) as client:
response: Final = await client.post(path, json=payload)
finally:
await runtime.stop(1)
assert response.status_code == 200, response.text
assert len(respx_mock.calls) == 1
if payload.get("background"):
assert response.json()["status"] == "in_progress"
else:
assert "hi" in response.text
assert not persisted.empty(), "Terminal usage must still be captured"
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_ack", [False, True])
async def test_text_completion_provider_rejection_captures_unknown_usage(
sync_ack: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
from fastapi import Request
from litellm.accounting import Outcome
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.db.accounting_ledger import UsageWrite
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
persisted: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
for write in writes:
persisted.put_nowait(write)
async def authorize(request: Request) -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="test-key", request_route=request.url.path)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setenv("OPENAI_API_KEY", "test-provider-key")
respx_mock.post().respond(
404,
json={
"error": {"message": "The model is unavailable", "type": "invalid_request_error", "code": "model_not_found"}
},
)
runtime: Final = UsageCaptureRuntime()
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": sync_ack})
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(
proxy_server,
"llm_router",
litellm.Router(
model_list=[
{
"model_name": "instruct",
"litellm_params": {"model": "openai/gpt-3.5-turbo-instruct", "api_key": "test-provider-key"},
}
],
num_retries=0,
),
)
monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(UserApiKeyCache()))
monkeypatch.setattr(litellm, "callbacks", [ProxyDBLogger()])
monkeypatch.setitem(proxy_server.app.dependency_overrides, proxy_server.user_api_key_auth, authorize)
runtime.start(persist, UsageCaptureSettings())
try:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=proxy_server.app), base_url="http://test"
) as client:
response: Final = await client.post(
"/v1/completions", json={"model": "instruct", "prompt": "hi", "max_tokens": 1}
)
finally:
await runtime.stop(1)
assert response.status_code == 404, response.text
assert len(respx_mock.calls) == 1
assert persisted.qsize() == 2
inbound: Final = persisted.get_nowait()
outbound: Final = persisted.get_nowait()
assert outbound.event.outcome is Outcome.FAILED
assert outbound.event.call_type.value == "atext_completion"
assert outbound.event.metrics is None
assert inbound.event.request_id == outbound.event.request_id
envelope: Final = json.loads(outbound.envelope)
assert envelope["usage"] is None
assert envelope["usage_source"] == "unknown"
assert envelope["pricing"]["reported_cost"] is None

View file

@ -4158,3 +4158,57 @@ def test_is_client_error_exception_existing_behaviour_unchanged() -> None:
ProxyException(message="x", type="x", param=None, code="429")
) is True
assert _is_client_error_exception(ValueError("x")) is False
@pytest.mark.asyncio
@pytest.mark.parametrize("tokens", [None, 0, 3])
async def test_failure_capture_preserves_prepared_usage_observation(tokens: int | None) -> None:
from litellm.proxy.db.accounting_ledger import UsageWrite
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_callback
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
timestamp: Final = real_datetime.datetime(2026, 1, 1)
committed: Final[asyncio.Queue[UsageWrite]] = asyncio.Queue()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
for write in writes:
committed.put_nowait(write)
logger: Final = Logging(
model=FAILURE_USAGE_MODEL,
messages=[],
stream=False,
call_type="acompletion",
start_time=timestamp,
litellm_call_id="post-response-failure",
function_id="post-response-failure",
)
logger.update_environment_variables(
model=FAILURE_USAGE_MODEL, custom_llm_provider="openai", litellm_params={}, optional_params={}
)
logger.model_call_details["response_cost"] = 0.2
runtime: Final = UsageCaptureRuntime()
runtime.start(persist, UsageCaptureSettings())
try:
prepared: Final = logger.prepare_response_cost(
{"id": "provider", "usage": None if tokens is None else {"prompt_tokens": tokens, "total_tokens": tokens}},
timestamp,
)
assert prepared is not None
lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logger}, provider_billed=True)
assert lifted["combined_usage_object"] is logger.model_call_details["combined_usage_object"]
assert lifted["response_cost"] == 0.2
assert runtime.meter is not None
capture_callback(
runtime.meter,
{**logger.model_call_details, **lifted, "log_event_type": "failed_api_call"},
None,
timestamp,
timestamp,
)
finally:
await runtime.stop(1)
committed.get_nowait()
envelope: Final = json.loads(committed.get_nowait().envelope)
assert (None if envelope["usage"] is None else envelope["usage"]["total_tokens"]) == tokens
assert envelope["usage_source"] == ("unknown" if tokens is None else "measured")

View file

@ -28644,7 +28644,7 @@ export interface components {
* CallTypes
* @enum {string}
*/
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill";
/** CallbackDelete */
CallbackDelete: {
/** Callback Name */
@ -29942,6 +29942,27 @@ export interface components {
* @description Documents all the fields supported by `general_settings` in config.yaml
*/
ConfigGeneralSettings: {
/** @description Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes. */
accounting_capture?: components["schemas"]["UsageCaptureSettings"] | null;
/**
* Accounting Sync Ack
* @description Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. Other requests remain supported with asynchronous usage capture.
* @default false
*/
accounting_sync_ack: boolean;
/**
* Accounting Sync Ack Failure Policy
* @description Return 503 or log the persistence failure after the response wait expires
* @default fail_closed
* @enum {string}
*/
accounting_sync_ack_failure_policy: "fail_closed" | "log";
/**
* Accounting Sync Ack Timeout Seconds
* @description Maximum response wait for usage commit, including the batch flush interval
* @default 20
*/
accounting_sync_ack_timeout_seconds: number;
/**
* Admission Queue Timeout Seconds
* @description maximum time a request waits for a worker slot
@ -49501,6 +49522,29 @@ export interface components {
/** User Role */
user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null;
};
/** UsageCaptureSettings */
UsageCaptureSettings: {
/**
* Batch Size
* @default 1000
*/
batch_size: number;
/**
* Flush Interval Seconds
* @default 5
*/
flush_interval_seconds: number;
/**
* Max Pending Bytes
* @default 16777216
*/
max_pending_bytes: number;
/**
* Max Pending Records
* @default 10000
*/
max_pending_records: number;
};
/** UsageChartPoint */
UsageChartPoint: {
/** Blocked */