mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(batches): store only chat usage extras on batch spend rows
Spend rows for batch retrieves copied OpenAI's batch fields (input_tokens, output_tokens and their details) into metadata.additional_usage_values next to the chat counts they duplicate. Build the row from the batch's chat usage, shared with the logging payload through batch_chat_usage.
This commit is contained in:
parent
2a852fa95b
commit
896d09b03e
4 changed files with 78 additions and 12 deletions
|
|
@ -20,7 +20,6 @@ from types import MappingProxyType, TracebackType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
|
||||
|
||||
from httpx import Response, Timeout
|
||||
from openai.types.batch_usage import BatchUsage
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
|
|
@ -200,6 +199,7 @@ from litellm.types.utils import (
|
|||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
batch_chat_usage,
|
||||
)
|
||||
from litellm.types.videos.main import VideoObject
|
||||
from litellm.utils import get_base_model_from_metadata, print_verbose
|
||||
|
|
@ -7253,15 +7253,6 @@ def _served_from_cache_at_no_charge(result: object) -> bool:
|
|||
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
|
||||
|
||||
|
||||
_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields)
|
||||
|
||||
|
||||
def _batch_chat_usage(response: object) -> Usage | None:
|
||||
if not isinstance(response, LiteLLMBatch) or response.usage is None:
|
||||
return None
|
||||
return Usage.model_validate(response.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS)))
|
||||
|
||||
|
||||
def _extract_response_obj_and_hidden_params(
|
||||
init_response_obj: object,
|
||||
original_exception: Exception | None,
|
||||
|
|
@ -7400,7 +7391,7 @@ def get_standard_logging_object_payload(
|
|||
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
|
||||
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
|
||||
or _batch_chat_usage(init_response_obj),
|
||||
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
|
||||
)
|
||||
usage_dict: Final = (
|
||||
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ from litellm.types.utils import (
|
|||
AzureSpillover,
|
||||
CallTypes,
|
||||
CostBreakdown,
|
||||
LiteLLMBatch,
|
||||
LlmProviders,
|
||||
StandardLoggingGuardrailInformation,
|
||||
StandardLoggingKubernetesPodRouting,
|
||||
|
|
@ -96,6 +97,7 @@ from litellm.types.utils import (
|
|||
StandardLoggingPayloadErrorInformation,
|
||||
StandardLoggingVectorStoreRequest,
|
||||
VectorStoreSearchResponse,
|
||||
batch_chat_usage,
|
||||
)
|
||||
from litellm.utils import get_end_user_id_for_cost_tracking
|
||||
|
||||
|
|
@ -618,7 +620,12 @@ def get_logging_payload(
|
|||
usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict)
|
||||
elif not is_unbilled_non_inference_call(call_type, metadata, response_obj_dict):
|
||||
# Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models
|
||||
_usage: Final = response_obj_dict.get("usage", None) or {}
|
||||
chat_usage_of_batch: Final = batch_chat_usage(response_obj) if isinstance(response_obj, LiteLLMBatch) else None
|
||||
_usage: Final = (
|
||||
chat_usage_of_batch.model_dump()
|
||||
if chat_usage_of_batch is not None
|
||||
else response_obj_dict.get("usage", None) or {}
|
||||
)
|
||||
if isinstance(_usage, litellm.Usage):
|
||||
usage = dict(_usage)
|
||||
elif isinstance(_usage, dict):
|
||||
|
|
|
|||
|
|
@ -4937,6 +4937,15 @@ class LiteLLMBatch(Batch):
|
|||
return self.dict()
|
||||
|
||||
|
||||
_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields)
|
||||
|
||||
|
||||
def batch_chat_usage(batch: LiteLLMBatch) -> Usage | None:
|
||||
if batch.usage is None:
|
||||
return None
|
||||
return Usage.model_validate(batch.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS)))
|
||||
|
||||
|
||||
class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase):
|
||||
# Events are already well-formed provider dicts. Validating them against the
|
||||
# OpenAIRealtimeEvents union makes Pydantic try every member per event, which
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
|
||||
LiteLLMBatch,
|
||||
StandardLoggingHiddenParams,
|
||||
StandardLoggingMetadata,
|
||||
StandardLoggingModelInformation,
|
||||
|
|
@ -222,6 +223,64 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token
|
|||
assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123
|
||||
|
||||
|
||||
_PROVIDER_READ_BATCH: Final = LiteLLMBatch.model_validate(
|
||||
{
|
||||
"id": "batch_spend_row",
|
||||
"completion_window": "24h",
|
||||
"created_at": 1,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"input_file_id": "file-in",
|
||||
"object": "batch",
|
||||
"status": "completed",
|
||||
"output_file_id": "file-out",
|
||||
"usage": {
|
||||
"input_tokens": 29,
|
||||
"input_tokens_details": {"cached_tokens": 8},
|
||||
"output_tokens": 46,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 75,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch",
|
||||
[
|
||||
_PROVIDER_READ_BATCH,
|
||||
_PROVIDER_READ_BATCH.model_copy(
|
||||
update={
|
||||
"usage": litellm.Usage(
|
||||
prompt_tokens=29,
|
||||
completion_tokens=46,
|
||||
total_tokens=75,
|
||||
prompt_tokens_details={"cached_tokens": 8},
|
||||
)
|
||||
}
|
||||
),
|
||||
],
|
||||
ids=["provider_read", "poller_billed"],
|
||||
)
|
||||
def test_batch_spend_row_stores_the_same_usage_extras_as_a_chat_call_with_its_counts(batch: LiteLLMBatch) -> None:
|
||||
payload: Final = get_logging_payload(
|
||||
kwargs={
|
||||
"model": "gpt-4o-mini",
|
||||
"call_type": "aretrieve_batch",
|
||||
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
|
||||
},
|
||||
response_obj=batch,
|
||||
start_time=datetime.datetime.now(timezone.utc),
|
||||
end_time=datetime.datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (29, 46, 75)
|
||||
assert json.loads(payload["metadata"])["additional_usage_values"] == _get_additional_usage_values_for_usage(
|
||||
litellm.Usage(
|
||||
prompt_tokens=29, completion_tokens=46, total_tokens=75, prompt_tokens_details={"cached_tokens": 8}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _HashingCache(litellm.Cache):
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue