fix(batches): store only chat usage extras on batch spend rows

Spend rows for batch retrieves copied OpenAI's batch fields (input_tokens,
output_tokens and their details) into metadata.additional_usage_values next
to the chat counts they duplicate. Build the row from the batch's chat usage,
shared with the logging payload through batch_chat_usage.
This commit is contained in:
mateo-berri 2026-10-29 16:20:06 -07:00
parent 2a852fa95b
commit 896d09b03e
4 changed files with 78 additions and 12 deletions

View file

@ -20,7 +20,6 @@ from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from httpx import Response, Timeout
from openai.types.batch_usage import BatchUsage
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -200,6 +199,7 @@ from litellm.types.utils import (
TextCompletionResponse,
TranscriptionResponse,
Usage,
batch_chat_usage,
)
from litellm.types.videos.main import VideoObject
from litellm.utils import get_base_model_from_metadata, print_verbose
@ -7253,15 +7253,6 @@ def _served_from_cache_at_no_charge(result: object) -> bool:
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields)
def _batch_chat_usage(response: object) -> Usage | None:
if not isinstance(response, LiteLLMBatch) or response.usage is None:
return None
return Usage.model_validate(response.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS)))
def _extract_response_obj_and_hidden_params(
init_response_obj: object,
original_exception: Exception | None,
@ -7400,7 +7391,7 @@ def get_standard_logging_object_payload(
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
or _batch_chat_usage(init_response_obj),
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
)
usage_dict: Final = (
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}

View file

@ -87,6 +87,7 @@ from litellm.types.utils import (
AzureSpillover,
CallTypes,
CostBreakdown,
LiteLLMBatch,
LlmProviders,
StandardLoggingGuardrailInformation,
StandardLoggingKubernetesPodRouting,
@ -96,6 +97,7 @@ from litellm.types.utils import (
StandardLoggingPayloadErrorInformation,
StandardLoggingVectorStoreRequest,
VectorStoreSearchResponse,
batch_chat_usage,
)
from litellm.utils import get_end_user_id_for_cost_tracking
@ -618,7 +620,12 @@ def get_logging_payload(
usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict)
elif not is_unbilled_non_inference_call(call_type, metadata, response_obj_dict):
# Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models
_usage: Final = response_obj_dict.get("usage", None) or {}
chat_usage_of_batch: Final = batch_chat_usage(response_obj) if isinstance(response_obj, LiteLLMBatch) else None
_usage: Final = (
chat_usage_of_batch.model_dump()
if chat_usage_of_batch is not None
else response_obj_dict.get("usage", None) or {}
)
if isinstance(_usage, litellm.Usage):
usage = dict(_usage)
elif isinstance(_usage, dict):

View file

@ -4937,6 +4937,15 @@ class LiteLLMBatch(Batch):
return self.dict()
_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields)
def batch_chat_usage(batch: LiteLLMBatch) -> Usage | None:
if batch.usage is None:
return None
return Usage.model_validate(batch.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS)))
class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase):
# Events are already well-formed provider dicts. Validating them against the
# OpenAIRealtimeEvents union makes Pydantic try every member per event, which

View file

@ -69,6 +69,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
LiteLLMBatch,
StandardLoggingHiddenParams,
StandardLoggingMetadata,
StandardLoggingModelInformation,
@ -222,6 +223,64 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token
assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123
_PROVIDER_READ_BATCH: Final = LiteLLMBatch.model_validate(
{
"id": "batch_spend_row",
"completion_window": "24h",
"created_at": 1,
"endpoint": "/v1/chat/completions",
"input_file_id": "file-in",
"object": "batch",
"status": "completed",
"output_file_id": "file-out",
"usage": {
"input_tokens": 29,
"input_tokens_details": {"cached_tokens": 8},
"output_tokens": 46,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": 75,
},
}
)
@pytest.mark.parametrize(
"batch",
[
_PROVIDER_READ_BATCH,
_PROVIDER_READ_BATCH.model_copy(
update={
"usage": litellm.Usage(
prompt_tokens=29,
completion_tokens=46,
total_tokens=75,
prompt_tokens_details={"cached_tokens": 8},
)
}
),
],
ids=["provider_read", "poller_billed"],
)
def test_batch_spend_row_stores_the_same_usage_extras_as_a_chat_call_with_its_counts(batch: LiteLLMBatch) -> None:
payload: Final = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"call_type": "aretrieve_batch",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=batch,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (29, 46, 75)
assert json.loads(payload["metadata"])["additional_usage_values"] == _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=29, completion_tokens=46, total_tokens=75, prompt_tokens_details={"cached_tokens": 8}
)
)
class _HashingCache(litellm.Cache):
def __init__(self) -> None:
pass