From 5dce13d2c24965111ea1178e992f701d922306b2 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 16:33:01 -0700 Subject: [PATCH] fix(proxy): attribute retry attempt spend rows to their deployment and keep them out of daily request counts (#47658) * fix(proxy): attribute retry attempt spend rows to their own deployment and keep them out of daily request counts Each -attempt- row now carries the model, provider, api_base and model group of the deployment that billed it (looked up by model_info.id through the proxy router, injected as deployment_lookup) instead of copying the final row's, and is stamped internal_call_origin=billed_retry_attempt so the daily spend tables keep its spend and tokens while counting zero api_requests and failed_requests for it * fix(proxy): declare DeploymentLookup as a TypeAlias and leave the test file's untouched lines as they were * fix(proxy): mark attempt rows with billed_retry_attempt instead of internal_call_origin so model usage keeps their spend Bugbot: an internal_call_origin stamp made build_model_usage_transaction drop the attempt rows from LiteLLM_DailyModelUsage. The rows now carry their own SpendLogsMetadata flag, which only the daily request counters read * fix(proxy): read the billed_retry_attempt flag through a typed helper so the basedpyright gate holds * fix(proxy): keep the typed litellm_params local in _deployment_provider so the basedpyright gate holds * fix(proxy): annotate the validated litellm_params as LiteLLM_Params in _deployment_provider * fix(proxy): keep billed retry attempts out of daily request errors and strip the flag from caller metadata Bugbot: _record_request_error counted every attempt row as a caller failure in LiteLLM_DailyRequestErrors. Veria: a caller could send metadata.billed_retry_attempt to zero its own request counts, so the key joins the untrusted metadata fields the proxy strips on the way in * fix(proxy): stamp billed_retry_attempt explicitly in the spend log metadata so the basedpyright gate holds --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/_types.py | 1 + litellm/proxy/db/db_spend_update_writer.py | 16 ++- .../proxy/hooks/proxy_track_cost_callback.py | 71 ++++++++-- litellm/proxy/litellm_pre_call_utils.py | 2 + .../spend_tracking/spend_tracking_utils.py | 13 +- .../gcs_pubsub/spend_logs_payload.json | 2 +- .../hooks/test_proxy_track_cost_callback.py | 123 ++++++++++++++++++ .../test_spend_management_endpoints.py | 2 +- 9 files changed, 216 insertions(+), 15 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 5b99a0f73dd..1c1dfb9aa1c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1654,6 +1654,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" +BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 97a2fc31fb0..54acf59ef80 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3731,6 +3731,7 @@ class SpendLogsMetadata(TypedDict): routing_decision: StandardLoggingRoutingDecision | None kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] internal_call_origin: InternalCallOrigin | None + billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index bb6ffe675c0..4e98823ba77 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -29,6 +29,7 @@ from litellm._internal_context import service_target, with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching import RedisCache from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -312,6 +313,11 @@ def _spend_tables_left_to_send( return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None +def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool: + """An earlier billed retry attempt's own spend row, not a request the caller sent.""" + return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True + + def _timed_request_duration_ms( payload: dict | SpendLogsPayload, request_status: Literal["success", "failure"], @@ -2805,11 +2811,12 @@ class DBSpendUpdateWriter: if not date: return metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults request_error_accumulator.record( payload=payload, request_status=prisma_client.get_request_status(payload), # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a typed spend payload date=date.split("T")[0], - is_internal_call=bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)), # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults + is_internal_call=internal_origin or _is_billed_retry_attempt(metadata), ) async def _common_add_spend_log_transaction_to_daily_transaction( @@ -2874,6 +2881,7 @@ class DBSpendUpdateWriter: endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) + caller_request: Final = not is_internal_call and not _is_billed_retry_attempt(_metadata) cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj) compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata) savings_spend: Final = compute_savings_spend( @@ -2908,9 +2916,9 @@ class DBSpendUpdateWriter: # requests the caller made: counting them inflates request-volume # readers, and an auto-router savings figure computed on a shadow # duplicate credits savings for traffic no user sent. - api_requests=0 if is_internal_call else 1, - successful_requests=1 if not is_internal_call and request_status == "success" else 0, - failed_requests=1 if not is_internal_call and request_status != "success" else 0, + api_requests=1 if caller_request else 0, + successful_requests=1 if caller_request and request_status == "success" else 0, + failed_requests=1 if caller_request and request_status != "success" else 0, cache_read_input_tokens=cache_read_input_tokens, cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj), compression_saved_tokens=compression_saved_tokens, diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 1f8b352fed9..994310c6619 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -5,14 +5,14 @@ from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Protocol, cast +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.batch_utils import batch_cost_is_final -from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED +from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( budget_reservation_from_metadata, @@ -54,11 +54,13 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import ( ) from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # legacy module exports _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + deployment_params_provider, get_request_model_access_groups, sanitize_error_information_for_spend_logs, should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend +from litellm.types.router import Deployment from litellm.types.utils import ( CallTypes, LiteLLMBatch, @@ -187,6 +189,14 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter: return proxy_logging_obj.db_spend_update_writer +DeploymentLookup: TypeAlias = Callable[[str], Deployment | None] + + +def _proxy_deployment(model_id: str) -> Deployment | None: + router: Final = get_llm_router() + return None if router is None else router.get_deployment(model_id=model_id) + + def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]: """The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any.""" attempts: Final = request_data.get("billed_failed_attempts") @@ -197,7 +207,10 @@ def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledF def _failure_spend_rows( - request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object + request_data: Mapping[str, object], + final_cost: float, + late_guardrails_snapshot: object, + deployment_lookup: DeploymentLookup, ) -> tuple[tuple[float, Mapping[str, object]], ...]: """One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the request id, then every earlier attempt the provider also billed under its own row.""" @@ -209,25 +222,59 @@ def _failure_spend_rows( return ( (final_cost, final_row), *( - (attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position)) + ( + attempt.response_cost, + _attempt_spend_row_kwargs( + request_data, attempt, position, _attempt_deployment(attempt, deployment_lookup) + ), + ) for position, attempt in enumerate(_billed_failed_attempts(request_data)) ), ) +def _attempt_deployment(attempt: BilledFailedAttempt, deployment_lookup: DeploymentLookup) -> Deployment | None: + return None if attempt.deployment_id is None else deployment_lookup(attempt.deployment_id) + + +def _deployment_attribution(deployment: Deployment) -> Mapping[str, object]: + """The spend-log attribution fields of the deployment that served an attempt: the model string it + was called with, its provider, its configured ``api_base`` and its model group.""" + return MappingProxyType( + { + "model": deployment.litellm_params.model, + "custom_llm_provider": deployment_params_provider(deployment.litellm_params), + "api_base": deployment.litellm_params.api_base, + "model_group": deployment.model_name, + } + ) + + +_NO_ATTRIBUTION: Final[Mapping[str, object]] = MappingProxyType({}) + + def _attempt_spend_row_kwargs( - request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int + request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int, deployment: Deployment | None ) -> Mapping[str, object]: """The final attempt's row kwargs re-keyed for one earlier billed attempt: its own ``request_id`` (``-attempt-``, since ``request_id`` is the spend-log primary key and the retry count restarts after a fallback hop), its estimated usage and cost, its retry - count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``.""" + count under ``attempted_retries``, the deployment that billed it under ``model_info.id`` and, + when the router still knows that deployment, its model, provider, ``api_base`` and model group. + The row is stamped ``billed_retry_attempt`` so the daily spend tables keep its spend, tokens + and model usage without counting it as a request the caller sent.""" suffix: Final = f"-attempt-{position}" + attribution: Final = _NO_ATTRIBUTION if deployment is None else _deployment_attribution(deployment) raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict - litellm_params: Final = dict(raw_params) + litellm_params: Final = { + **raw_params, + **({} if deployment is None else {"api_base": attribution["api_base"]}), + } metadata: Final = { **cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict "attempted_retries": attempt.attempted_retries, + BILLED_RETRY_ATTEMPT_METADATA_KEY: True, + **({} if deployment is None else {"model_group": attribution["model_group"]}), } metadata.pop("previous_models", None) if attempt.deployment_id is not None: @@ -238,6 +285,7 @@ def _attempt_spend_row_kwargs( litellm_params["metadata"] = metadata row: Final = { **request_data, + **({} if deployment is None else {k: attribution[k] for k in ("model", "custom_llm_provider")}), "litellm_params": litellm_params, "combined_usage_object": attempt.usage, "response_cost": attempt.response_cost, @@ -250,7 +298,11 @@ def _attempt_spend_row_kwargs( if isinstance(standard_logging_object, dict): payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict if isinstance(payload.get("id"), str): - row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"} + row["standard_logging_object"] = { + **payload, + "id": f"{payload['id']}{suffix}", + **({} if deployment is None else {**attribution, "model_id": attempt.deployment_id}), + } return row @@ -262,10 +314,12 @@ class _ProxyDBLogger(CustomLogger): turn_off_message_logging: bool = False, message_logging: bool = True, spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer, + deployment_lookup: DeploymentLookup = _proxy_deployment, ) -> None: super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging) self.spend_event_producer = spend_event_producer self._spend_writer: Final = spend_writer + self._deployment_lookup: Final = deployment_lookup async def async_log_success_event( self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime @@ -439,6 +493,7 @@ class _ProxyDBLogger(CustomLogger): cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below recovered_response_cost, failure_snapshot_with_late_guardrails, + self._deployment_lookup, ), ) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3ac270fcf2c..09f53cecb72 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -29,6 +29,7 @@ from litellm._service_logger import ServiceLogging from litellm._uuid import uuid from litellm.constants import ( BAGGAGE_MAPPED_TAGS_METADATA_KEY, + BILLED_RETRY_ATTEMPT_METADATA_KEY, CALLER_SESSION_ID_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, @@ -426,6 +427,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, + BILLED_RETRY_ATTEMPT_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, "standard_logging_object", "litellm_roi_estimator", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 091fb5d1305..62aee788ed0 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, CALLER_SESSION_ID_METADATA_KEY, CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, @@ -187,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) ) @@ -249,6 +251,7 @@ def _get_spend_logs_metadata( guardrail_information=None, internal_call_origin=None, litellm_roi_estimator=False, + billed_retry_attempt=False, eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, @@ -270,6 +273,8 @@ def _get_spend_logs_metadata( ) verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata]) + retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY + billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True # Filter the metadata dictionary to include only the specified keys clean_metadata: Final = SpendLogsMetadata( **MappingProxyType( @@ -283,6 +288,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) _raw_key: Final = clean_metadata.get("user_api_key") @@ -453,7 +459,12 @@ def _sl_attribution_fallback( def _deployment_provider(deployment: DeploymentTypedDict) -> str | None: - litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"]) + litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(deployment["litellm_params"]) + return deployment_params_provider(litellm_params) + + +def deployment_params_provider(litellm_params: LiteLLM_Params) -> str | None: + """The provider a deployment's ``litellm_params`` authenticate against, None when its model cannot be resolved.""" if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params): return LlmProviders.LITELLM_PROXY.value declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider) diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index cfdf707c977..6e5461c3b60 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index ba1b5536a07..bd997f10322 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -16,6 +16,7 @@ from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.collector import SpendEventConsumer from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( _get_budget_reservation_from_metadata, @@ -29,6 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpend from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage @@ -154,6 +156,127 @@ async def test_each_earlier_billed_attempt_gets_its_own_spend_row(): assert row["team_id"] == "t1" +def _deployment(model_name: str, model_id: str, model: str, api_base: str | None) -> Deployment: + return Deployment( + model_name=model_name, + litellm_params=LiteLLM_Params(model=model, api_base=api_base), + model_info={"id": model_id}, + ) + + +_TIMED_OUT_AT: Final = datetime(2026, 10, 20, 21, 24, 48, tzinfo=timezone.utc) + + +async def _spend_log_payloads_of_a_retried_timeout_across_a_fallback() -> list[SpendLogsPayload]: + """The spend-log payloads a timed-out request writes when attempt 0 and 1 ran on the anthropic + deployment of group ``essay`` and the request then fell back to the openai deployment of group + ``essay-fallback``: the final row first, then one row per earlier billed attempt.""" + from litellm.proxy.utils import BilledFailedAttempt + + deployments = { + "dep-anthropic": _deployment("essay", "dep-anthropic", "anthropic/claude-sonnet-4-6", None), + "dep-openai": _deployment("essay-fallback", "dep-openai", "openai/gpt-4.1", "https://api.openai.com/v1"), + } + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + request_data = { + "model": "anthropic/claude-sonnet-4-6", + "custom_llm_provider": "anthropic", + "litellm_call_id": "call-1", + "litellm_params": {"api_base": "https://api.anthropic.com/v1/messages"}, + "standard_logging_object": { + "id": "call-1", + "model": "anthropic/claude-sonnet-4-6", + "custom_llm_provider": "anthropic", + "api_base": "https://api.anthropic.com/v1/messages", + "model_id": "dep-anthropic", + "model_group": "essay", + "metadata": {}, + "hidden_params": {}, + "model_map_information": {}, + }, + "metadata": {"attempted_retries": 1, "model_group": "essay", "model_info": {"id": "dep-anthropic"}}, + "combined_usage_object": usage, + "response_cost": 7.2e-05, + "billed_failed_attempts": ( + BilledFailedAttempt(attempted_retries=1, deployment_id="dep-anthropic", usage=usage, response_cost=7.2e-05), + BilledFailedAttempt(attempted_retries=0, deployment_id="dep-openai", usage=usage, response_cost=4.8e-05), + ), + } + exception = litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai") + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await ProxyDBLogger(deployment_lookup=deployments.get).async_post_call_failure_hook( + request_data=request_data, + original_exception=exception, + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", user_id="u1"), + ) + return [ + get_logging_payload( + kwargs=call.kwargs["kwargs"], + response_obj=exception, + start_time=_TIMED_OUT_AT, + end_time=_TIMED_OUT_AT + timedelta(seconds=11), + ) + for call in mock_update_database.call_args_list + ] + + +@pytest.mark.asyncio +async def test_attempt_rows_are_attributed_to_the_deployment_that_billed_them(): + """Regression since #47416: an earlier attempt that ran on another deployment was written with the + final row's model, provider and api_base. Each attempt row names the deployment it ran on.""" + final_row, attempt_0, attempt_1 = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback() + + assert (final_row["model"], final_row["custom_llm_provider"], final_row["model_id"]) == ( + "anthropic/claude-sonnet-4-6", + "anthropic", + "dep-anthropic", + ) + assert final_row["api_base"] == "https://api.anthropic.com/v1/messages" + assert (attempt_0["model"], attempt_0["custom_llm_provider"], attempt_0["model_id"], attempt_0["model_group"]) == ( + "anthropic/claude-sonnet-4-6", + "anthropic", + "dep-anthropic", + "essay", + ) + assert (attempt_1["model"], attempt_1["custom_llm_provider"], attempt_1["model_id"], attempt_1["model_group"]) == ( + "openai/gpt-4.1", + "openai", + "dep-openai", + "essay-fallback", + ) + assert attempt_1["api_base"] == "https://api.openai.com/v1" + assert attempt_1["request_id"] == "call-1-attempt-1" + assert attempt_1["spend"] == 4.8e-05 + + +@pytest.mark.asyncio +async def test_attempt_rows_add_spend_but_not_requests_to_daily_spend(): + """Regression since #47416: one caller request with N billed earlier attempts counted N+1 requests in + the daily spend tables. The attempt rows keep their spend, tokens and model usage and count zero requests.""" + payloads = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback() + prisma = MagicMock() + prisma.get_request_status = MagicMock(return_value="failure") + + daily = [ + await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction( + payload=payload, prisma_client=prisma, type="user" + ) + for payload in payloads + ] + + assert all(row is not None for row in daily) + assert [row["api_requests"] for row in daily if row is not None] == [1, 0, 0] + assert [row["failed_requests"] for row in daily if row is not None] == [1, 0, 0] + assert [row["spend"] for row in daily if row is not None] == [7.2e-05, 7.2e-05, 4.8e-05] + assert [row["prompt_tokens"] for row in daily if row is not None] == [24, 24, 24] + model_usage = [build_model_usage_transaction(payload) for payload in payloads] + assert all(usage is not None for usage in model_usage), "attempt spend must still reach daily model usage" + assert [usage.spend for usage in model_usage if usage is not None] == [7.2e-05, 7.2e-05, 4.8e-05] + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata(): """ diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index ed8ae2db8b1..23bbe521670 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3725,7 +3725,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30,