mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(proxy): attribute retry attempt spend rows to their deployment and keep them out of daily request counts (#47658)
* fix(proxy): attribute retry attempt spend rows to their own deployment and keep them out of daily request counts Each <request_id>-attempt-<n> row now carries the model, provider, api_base and model group of the deployment that billed it (looked up by model_info.id through the proxy router, injected as deployment_lookup) instead of copying the final row's, and is stamped internal_call_origin=billed_retry_attempt so the daily spend tables keep its spend and tokens while counting zero api_requests and failed_requests for it * fix(proxy): declare DeploymentLookup as a TypeAlias and leave the test file's untouched lines as they were * fix(proxy): mark attempt rows with billed_retry_attempt instead of internal_call_origin so model usage keeps their spend Bugbot: an internal_call_origin stamp made build_model_usage_transaction drop the attempt rows from LiteLLM_DailyModelUsage. The rows now carry their own SpendLogsMetadata flag, which only the daily request counters read * fix(proxy): read the billed_retry_attempt flag through a typed helper so the basedpyright gate holds * fix(proxy): keep the typed litellm_params local in _deployment_provider so the basedpyright gate holds * fix(proxy): annotate the validated litellm_params as LiteLLM_Params in _deployment_provider * fix(proxy): keep billed retry attempts out of daily request errors and strip the flag from caller metadata Bugbot: _record_request_error counted every attempt row as a caller failure in LiteLLM_DailyRequestErrors. Veria: a caller could send metadata.billed_retry_attempt to zero its own request counts, so the key joins the untrusted metadata fields the proxy strips on the way in * fix(proxy): stamp billed_retry_attempt explicitly in the spend log metadata so the basedpyright gate holds --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu
GitHub
parent
f3e1307ed9
commit
5dce13d2c2
9 changed files with 216 additions and 15 deletions
|
|
@ -1654,6 +1654,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
|||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
|
||||
|
|
|
|||
|
|
@ -3731,6 +3731,7 @@ class SpendLogsMetadata(TypedDict):
|
|||
routing_decision: StandardLoggingRoutingDecision | None
|
||||
kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None]
|
||||
internal_call_origin: InternalCallOrigin | None
|
||||
billed_retry_attempt: ReadOnly[NotRequired[bool | None]]
|
||||
litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
|
||||
guardrail_information: list[StandardLoggingGuardrailInformation] | None
|
||||
eval_information: Any | None
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm._internal_context import service_target, with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching import RedisCache
|
||||
from litellm.constants import (
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
|
||||
DB_SPEND_UPDATE_JOB_NAME,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
|
|
@ -312,6 +313,11 @@ def _spend_tables_left_to_send(
|
|||
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
|
||||
|
||||
|
||||
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
|
||||
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
|
||||
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
|
||||
|
||||
|
||||
def _timed_request_duration_ms(
|
||||
payload: dict | SpendLogsPayload,
|
||||
request_status: Literal["success", "failure"],
|
||||
|
|
@ -2805,11 +2811,12 @@ class DBSpendUpdateWriter:
|
|||
if not date:
|
||||
return
|
||||
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
|
||||
request_error_accumulator.record(
|
||||
payload=payload,
|
||||
request_status=prisma_client.get_request_status(payload), # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a typed spend payload
|
||||
date=date.split("T")[0],
|
||||
is_internal_call=bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)), # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
|
||||
is_internal_call=internal_origin or _is_billed_retry_attempt(metadata),
|
||||
)
|
||||
|
||||
async def _common_add_spend_log_transaction_to_daily_transaction(
|
||||
|
|
@ -2874,6 +2881,7 @@ class DBSpendUpdateWriter:
|
|||
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
|
||||
|
||||
is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
|
||||
caller_request: Final = not is_internal_call and not _is_billed_retry_attempt(_metadata)
|
||||
cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj)
|
||||
compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata)
|
||||
savings_spend: Final = compute_savings_spend(
|
||||
|
|
@ -2908,9 +2916,9 @@ class DBSpendUpdateWriter:
|
|||
# requests the caller made: counting them inflates request-volume
|
||||
# readers, and an auto-router savings figure computed on a shadow
|
||||
# duplicate credits savings for traffic no user sent.
|
||||
api_requests=0 if is_internal_call else 1,
|
||||
successful_requests=1 if not is_internal_call and request_status == "success" else 0,
|
||||
failed_requests=1 if not is_internal_call and request_status != "success" else 0,
|
||||
api_requests=1 if caller_request else 0,
|
||||
successful_requests=1 if caller_request and request_status == "success" else 0,
|
||||
failed_requests=1 if caller_request and request_status != "success" else 0,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj),
|
||||
compression_saved_tokens=compression_saved_tokens,
|
||||
|
|
|
|||
|
|
@ -5,14 +5,14 @@ from contextlib import contextmanager
|
|||
from contextvars import ContextVar
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Protocol, cast
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.batches.batch_utils import batch_cost_is_final
|
||||
from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED
|
||||
from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
budget_reservation_from_metadata,
|
||||
|
|
@ -54,11 +54,13 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # legacy module exports
|
||||
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
deployment_params_provider,
|
||||
get_request_model_access_groups,
|
||||
sanitize_error_information_for_spend_logs,
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
)
|
||||
from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
LiteLLMBatch,
|
||||
|
|
@ -187,6 +189,14 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
|
|||
return proxy_logging_obj.db_spend_update_writer
|
||||
|
||||
|
||||
DeploymentLookup: TypeAlias = Callable[[str], Deployment | None]
|
||||
|
||||
|
||||
def _proxy_deployment(model_id: str) -> Deployment | None:
|
||||
router: Final = get_llm_router()
|
||||
return None if router is None else router.get_deployment(model_id=model_id)
|
||||
|
||||
|
||||
def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
|
||||
"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
|
||||
attempts: Final = request_data.get("billed_failed_attempts")
|
||||
|
|
@ -197,7 +207,10 @@ def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledF
|
|||
|
||||
|
||||
def _failure_spend_rows(
|
||||
request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object
|
||||
request_data: Mapping[str, object],
|
||||
final_cost: float,
|
||||
late_guardrails_snapshot: object,
|
||||
deployment_lookup: DeploymentLookup,
|
||||
) -> tuple[tuple[float, Mapping[str, object]], ...]:
|
||||
"""One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the
|
||||
request id, then every earlier attempt the provider also billed under its own row."""
|
||||
|
|
@ -209,25 +222,59 @@ def _failure_spend_rows(
|
|||
return (
|
||||
(final_cost, final_row),
|
||||
*(
|
||||
(attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))
|
||||
(
|
||||
attempt.response_cost,
|
||||
_attempt_spend_row_kwargs(
|
||||
request_data, attempt, position, _attempt_deployment(attempt, deployment_lookup)
|
||||
),
|
||||
)
|
||||
for position, attempt in enumerate(_billed_failed_attempts(request_data))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _attempt_deployment(attempt: BilledFailedAttempt, deployment_lookup: DeploymentLookup) -> Deployment | None:
|
||||
return None if attempt.deployment_id is None else deployment_lookup(attempt.deployment_id)
|
||||
|
||||
|
||||
def _deployment_attribution(deployment: Deployment) -> Mapping[str, object]:
|
||||
"""The spend-log attribution fields of the deployment that served an attempt: the model string it
|
||||
was called with, its provider, its configured ``api_base`` and its model group."""
|
||||
return MappingProxyType(
|
||||
{
|
||||
"model": deployment.litellm_params.model,
|
||||
"custom_llm_provider": deployment_params_provider(deployment.litellm_params),
|
||||
"api_base": deployment.litellm_params.api_base,
|
||||
"model_group": deployment.model_name,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_NO_ATTRIBUTION: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _attempt_spend_row_kwargs(
|
||||
request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int
|
||||
request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int, deployment: Deployment | None
|
||||
) -> Mapping[str, object]:
|
||||
"""The final attempt's row kwargs re-keyed for one earlier billed attempt: its own
|
||||
``request_id`` (``<id>-attempt-<position>``, since ``request_id`` is the spend-log primary key
|
||||
and the retry count restarts after a fallback hop), its estimated usage and cost, its retry
|
||||
count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""
|
||||
count under ``attempted_retries``, the deployment that billed it under ``model_info.id`` and,
|
||||
when the router still knows that deployment, its model, provider, ``api_base`` and model group.
|
||||
The row is stamped ``billed_retry_attempt`` so the daily spend tables keep its spend, tokens
|
||||
and model usage without counting it as a request the caller sent."""
|
||||
suffix: Final = f"-attempt-{position}"
|
||||
attribution: Final = _NO_ATTRIBUTION if deployment is None else _deployment_attribution(deployment)
|
||||
raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict
|
||||
litellm_params: Final = dict(raw_params)
|
||||
litellm_params: Final = {
|
||||
**raw_params,
|
||||
**({} if deployment is None else {"api_base": attribution["api_base"]}),
|
||||
}
|
||||
metadata: Final = {
|
||||
**cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict
|
||||
"attempted_retries": attempt.attempted_retries,
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: True,
|
||||
**({} if deployment is None else {"model_group": attribution["model_group"]}),
|
||||
}
|
||||
metadata.pop("previous_models", None)
|
||||
if attempt.deployment_id is not None:
|
||||
|
|
@ -238,6 +285,7 @@ def _attempt_spend_row_kwargs(
|
|||
litellm_params["metadata"] = metadata
|
||||
row: Final = {
|
||||
**request_data,
|
||||
**({} if deployment is None else {k: attribution[k] for k in ("model", "custom_llm_provider")}),
|
||||
"litellm_params": litellm_params,
|
||||
"combined_usage_object": attempt.usage,
|
||||
"response_cost": attempt.response_cost,
|
||||
|
|
@ -250,7 +298,11 @@ def _attempt_spend_row_kwargs(
|
|||
if isinstance(standard_logging_object, dict):
|
||||
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
|
||||
if isinstance(payload.get("id"), str):
|
||||
row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}
|
||||
row["standard_logging_object"] = {
|
||||
**payload,
|
||||
"id": f"{payload['id']}{suffix}",
|
||||
**({} if deployment is None else {**attribution, "model_id": attempt.deployment_id}),
|
||||
}
|
||||
return row
|
||||
|
||||
|
||||
|
|
@ -262,10 +314,12 @@ class _ProxyDBLogger(CustomLogger):
|
|||
turn_off_message_logging: bool = False,
|
||||
message_logging: bool = True,
|
||||
spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer,
|
||||
deployment_lookup: DeploymentLookup = _proxy_deployment,
|
||||
) -> None:
|
||||
super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging)
|
||||
self.spend_event_producer = spend_event_producer
|
||||
self._spend_writer: Final = spend_writer
|
||||
self._deployment_lookup: Final = deployment_lookup
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
|
||||
|
|
@ -439,6 +493,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below
|
||||
recovered_response_cost,
|
||||
failure_snapshot_with_late_guardrails,
|
||||
self._deployment_lookup,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm._service_logger import ServiceLogging
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BAGGAGE_MAPPED_TAGS_METADATA_KEY,
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
CLIENT_OUTPUT_CEILING_METADATA_KEY,
|
||||
CONSUMED_REQUEST_TAGS_METADATA_KEY,
|
||||
|
|
@ -426,6 +427,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
|
|||
CONSUMED_REQUEST_TAGS_METADATA_KEY,
|
||||
ROUTING_REQUEST_TAGS_METADATA_KEY,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
|
||||
"standard_logging_object",
|
||||
"litellm_roi_estimator",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import (
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
CALLER_SESSION_ID_METADATA_KEY,
|
||||
CLI_SESSION_KEY_PREFIX,
|
||||
EMPTY_MAPPING,
|
||||
|
|
@ -187,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
|
|||
"used_client_oauth_token",
|
||||
KUBERNETES_POD_ROUTING_KEY,
|
||||
"litellm_roi_estimator",
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY,
|
||||
"user_api_key_user_email",
|
||||
)
|
||||
)
|
||||
|
|
@ -249,6 +251,7 @@ def _get_spend_logs_metadata(
|
|||
guardrail_information=None,
|
||||
internal_call_origin=None,
|
||||
litellm_roi_estimator=False,
|
||||
billed_retry_attempt=False,
|
||||
eval_information=None,
|
||||
cold_storage_object_key=cold_storage_object_key,
|
||||
litellm_overhead_time_ms=None,
|
||||
|
|
@ -270,6 +273,8 @@ def _get_spend_logs_metadata(
|
|||
)
|
||||
verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata])
|
||||
|
||||
retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY
|
||||
billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True
|
||||
# Filter the metadata dictionary to include only the specified keys
|
||||
clean_metadata: Final = SpendLogsMetadata(
|
||||
**MappingProxyType(
|
||||
|
|
@ -283,6 +288,7 @@ def _get_spend_logs_metadata(
|
|||
used_client_oauth_token=used_client_oauth_token,
|
||||
kubernetes_pod_routing=kubernetes_pod_routing,
|
||||
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
|
||||
billed_retry_attempt=billed_retry_attempt,
|
||||
user_api_key_user_email=_stored_user_email(metadata),
|
||||
)
|
||||
_raw_key: Final = clean_metadata.get("user_api_key")
|
||||
|
|
@ -453,7 +459,12 @@ def _sl_attribution_fallback(
|
|||
|
||||
|
||||
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
|
||||
litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"])
|
||||
litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(deployment["litellm_params"])
|
||||
return deployment_params_provider(litellm_params)
|
||||
|
||||
|
||||
def deployment_params_provider(litellm_params: LiteLLM_Params) -> str | None:
|
||||
"""The provider a deployment's ``litellm_params`` authenticate against, None when its model cannot be resolved."""
|
||||
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params):
|
||||
return LlmProviders.LITELLM_PROXY.value
|
||||
declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider)
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
"user": "",
|
||||
"team_id": "",
|
||||
"organization_id": "",
|
||||
"metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
|
||||
"metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth
|
|||
from litellm.proxy.collector import SpendEventConsumer
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction
|
||||
from litellm.proxy.db.spend_log_tool_index import response_tool_call_names
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import (
|
||||
_get_budget_reservation_from_metadata,
|
||||
|
|
@ -29,6 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
|
|||
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
|
||||
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
|
||||
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
|
||||
from litellm.types.router import Deployment, LiteLLM_Params
|
||||
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
|
||||
|
|
@ -154,6 +156,127 @@ async def test_each_earlier_billed_attempt_gets_its_own_spend_row():
|
|||
assert row["team_id"] == "t1"
|
||||
|
||||
|
||||
def _deployment(model_name: str, model_id: str, model: str, api_base: str | None) -> Deployment:
|
||||
return Deployment(
|
||||
model_name=model_name,
|
||||
litellm_params=LiteLLM_Params(model=model, api_base=api_base),
|
||||
model_info={"id": model_id},
|
||||
)
|
||||
|
||||
|
||||
_TIMED_OUT_AT: Final = datetime(2026, 10, 20, 21, 24, 48, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
async def _spend_log_payloads_of_a_retried_timeout_across_a_fallback() -> list[SpendLogsPayload]:
|
||||
"""The spend-log payloads a timed-out request writes when attempt 0 and 1 ran on the anthropic
|
||||
deployment of group ``essay`` and the request then fell back to the openai deployment of group
|
||||
``essay-fallback``: the final row first, then one row per earlier billed attempt."""
|
||||
from litellm.proxy.utils import BilledFailedAttempt
|
||||
|
||||
deployments = {
|
||||
"dep-anthropic": _deployment("essay", "dep-anthropic", "anthropic/claude-sonnet-4-6", None),
|
||||
"dep-openai": _deployment("essay-fallback", "dep-openai", "openai/gpt-4.1", "https://api.openai.com/v1"),
|
||||
}
|
||||
usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
|
||||
request_data = {
|
||||
"model": "anthropic/claude-sonnet-4-6",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"litellm_call_id": "call-1",
|
||||
"litellm_params": {"api_base": "https://api.anthropic.com/v1/messages"},
|
||||
"standard_logging_object": {
|
||||
"id": "call-1",
|
||||
"model": "anthropic/claude-sonnet-4-6",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"api_base": "https://api.anthropic.com/v1/messages",
|
||||
"model_id": "dep-anthropic",
|
||||
"model_group": "essay",
|
||||
"metadata": {},
|
||||
"hidden_params": {},
|
||||
"model_map_information": {},
|
||||
},
|
||||
"metadata": {"attempted_retries": 1, "model_group": "essay", "model_info": {"id": "dep-anthropic"}},
|
||||
"combined_usage_object": usage,
|
||||
"response_cost": 7.2e-05,
|
||||
"billed_failed_attempts": (
|
||||
BilledFailedAttempt(attempted_retries=1, deployment_id="dep-anthropic", usage=usage, response_cost=7.2e-05),
|
||||
BilledFailedAttempt(attempted_retries=0, deployment_id="dep-openai", usage=usage, response_cost=4.8e-05),
|
||||
),
|
||||
}
|
||||
exception = litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai")
|
||||
with patch(
|
||||
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_update_database:
|
||||
await ProxyDBLogger(deployment_lookup=deployments.get).async_post_call_failure_hook(
|
||||
request_data=request_data,
|
||||
original_exception=exception,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", user_id="u1"),
|
||||
)
|
||||
return [
|
||||
get_logging_payload(
|
||||
kwargs=call.kwargs["kwargs"],
|
||||
response_obj=exception,
|
||||
start_time=_TIMED_OUT_AT,
|
||||
end_time=_TIMED_OUT_AT + timedelta(seconds=11),
|
||||
)
|
||||
for call in mock_update_database.call_args_list
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_attempt_rows_are_attributed_to_the_deployment_that_billed_them():
|
||||
"""Regression since #47416: an earlier attempt that ran on another deployment was written with the
|
||||
final row's model, provider and api_base. Each attempt row names the deployment it ran on."""
|
||||
final_row, attempt_0, attempt_1 = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback()
|
||||
|
||||
assert (final_row["model"], final_row["custom_llm_provider"], final_row["model_id"]) == (
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"anthropic",
|
||||
"dep-anthropic",
|
||||
)
|
||||
assert final_row["api_base"] == "https://api.anthropic.com/v1/messages"
|
||||
assert (attempt_0["model"], attempt_0["custom_llm_provider"], attempt_0["model_id"], attempt_0["model_group"]) == (
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"anthropic",
|
||||
"dep-anthropic",
|
||||
"essay",
|
||||
)
|
||||
assert (attempt_1["model"], attempt_1["custom_llm_provider"], attempt_1["model_id"], attempt_1["model_group"]) == (
|
||||
"openai/gpt-4.1",
|
||||
"openai",
|
||||
"dep-openai",
|
||||
"essay-fallback",
|
||||
)
|
||||
assert attempt_1["api_base"] == "https://api.openai.com/v1"
|
||||
assert attempt_1["request_id"] == "call-1-attempt-1"
|
||||
assert attempt_1["spend"] == 4.8e-05
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_attempt_rows_add_spend_but_not_requests_to_daily_spend():
|
||||
"""Regression since #47416: one caller request with N billed earlier attempts counted N+1 requests in
|
||||
the daily spend tables. The attempt rows keep their spend, tokens and model usage and count zero requests."""
|
||||
payloads = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback()
|
||||
prisma = MagicMock()
|
||||
prisma.get_request_status = MagicMock(return_value="failure")
|
||||
|
||||
daily = [
|
||||
await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction(
|
||||
payload=payload, prisma_client=prisma, type="user"
|
||||
)
|
||||
for payload in payloads
|
||||
]
|
||||
|
||||
assert all(row is not None for row in daily)
|
||||
assert [row["api_requests"] for row in daily if row is not None] == [1, 0, 0]
|
||||
assert [row["failed_requests"] for row in daily if row is not None] == [1, 0, 0]
|
||||
assert [row["spend"] for row in daily if row is not None] == [7.2e-05, 7.2e-05, 4.8e-05]
|
||||
assert [row["prompt_tokens"] for row in daily if row is not None] == [24, 24, 24]
|
||||
model_usage = [build_model_usage_transaction(payload) for payload in payloads]
|
||||
assert all(usage is not None for usage in model_usage), "attempt spend must still reach daily model usage"
|
||||
assert [usage.spend for usage in model_usage if usage is not None] == [7.2e-05, 7.2e-05, 4.8e-05]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3725,7 +3725,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue