From 896d09b03ed0b0ada10bb53ffd0b0f1fb30ebf96 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:20:06 -0700 Subject: [PATCH] fix(batches): store only chat usage extras on batch spend rows Spend rows for batch retrieves copied OpenAI's batch fields (input_tokens, output_tokens and their details) into metadata.additional_usage_values next to the chat counts they duplicate. Build the row from the batch's chat usage, shared with the logging payload through batch_chat_usage. --- litellm/litellm_core_utils/litellm_logging.py | 13 +--- .../spend_tracking/spend_tracking_utils.py | 9 ++- litellm/types/utils.py | 9 +++ .../test_spend_tracking_utils.py | 59 +++++++++++++++++++ 4 files changed, 78 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a699e09a512..412679fa1d0 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -20,7 +20,6 @@ from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast from httpx import Response, Timeout -from openai.types.batch_usage import BatchUsage from pydantic import BaseModel, JsonValue, TypeAdapter import litellm @@ -200,6 +199,7 @@ from litellm.types.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, + batch_chat_usage, ) from litellm.types.videos.main import VideoObject from litellm.utils import get_base_model_from_metadata, print_verbose @@ -7253,15 +7253,6 @@ def _served_from_cache_at_no_charge(result: object) -> bool: return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0 -_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields) - - -def _batch_chat_usage(response: object) -> Usage | None: - if not isinstance(response, LiteLLMBatch) or response.usage is None: - return None - return Usage.model_validate(response.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS))) - - def _extract_response_obj_and_hidden_params( init_response_obj: object, original_exception: Exception | None, @@ -7400,7 +7391,7 @@ def get_standard_logging_object_payload( raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict( response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj, combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")) - or _batch_chat_usage(init_response_obj), + or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None), ) usage_dict: Final = ( {**raw_usage_dict, "output_image_count": len(init_response_obj.data)} diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..6b234e42847 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -87,6 +87,7 @@ from litellm.types.utils import ( AzureSpillover, CallTypes, CostBreakdown, + LiteLLMBatch, LlmProviders, StandardLoggingGuardrailInformation, StandardLoggingKubernetesPodRouting, @@ -96,6 +97,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, StandardLoggingVectorStoreRequest, VectorStoreSearchResponse, + batch_chat_usage, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -618,7 +620,12 @@ def get_logging_payload( usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) elif not is_unbilled_non_inference_call(call_type, metadata, response_obj_dict): # Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models - _usage: Final = response_obj_dict.get("usage", None) or {} + chat_usage_of_batch: Final = batch_chat_usage(response_obj) if isinstance(response_obj, LiteLLMBatch) else None + _usage: Final = ( + chat_usage_of_batch.model_dump() + if chat_usage_of_batch is not None + else response_obj_dict.get("usage", None) or {} + ) if isinstance(_usage, litellm.Usage): usage = dict(_usage) elif isinstance(_usage, dict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7ba63caa179..5cd95f29894 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4937,6 +4937,15 @@ class LiteLLMBatch(Batch): return self.dict() +_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields) + + +def batch_chat_usage(batch: LiteLLMBatch) -> Usage | None: + if batch.usage is None: + return None + return Usage.model_validate(batch.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS))) + + class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase): # Events are already well-formed provider dicts. Validating them against the # OpenAIRealtimeEvents union makes Pydantic try every member per event, which diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..600a56a1128 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -69,6 +69,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + LiteLLMBatch, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, @@ -222,6 +223,64 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123 +_PROVIDER_READ_BATCH: Final = LiteLLMBatch.model_validate( + { + "id": "batch_spend_row", + "completion_window": "24h", + "created_at": 1, + "endpoint": "/v1/chat/completions", + "input_file_id": "file-in", + "object": "batch", + "status": "completed", + "output_file_id": "file-out", + "usage": { + "input_tokens": 29, + "input_tokens_details": {"cached_tokens": 8}, + "output_tokens": 46, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 75, + }, + } +) + + +@pytest.mark.parametrize( + "batch", + [ + _PROVIDER_READ_BATCH, + _PROVIDER_READ_BATCH.model_copy( + update={ + "usage": litellm.Usage( + prompt_tokens=29, + completion_tokens=46, + total_tokens=75, + prompt_tokens_details={"cached_tokens": 8}, + ) + } + ), + ], + ids=["provider_read", "poller_billed"], +) +def test_batch_spend_row_stores_the_same_usage_extras_as_a_chat_call_with_its_counts(batch: LiteLLMBatch) -> None: + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "call_type": "aretrieve_batch", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=batch, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (29, 46, 75) + assert json.loads(payload["metadata"])["additional_usage_values"] == _get_additional_usage_values_for_usage( + litellm.Usage( + prompt_tokens=29, completion_tokens=46, total_tokens=75, prompt_tokens_details={"cached_tokens": 8} + ) + ) + + class _HashingCache(litellm.Cache): def __init__(self) -> None: pass