fix(batches): keep sync logger events at $0 on poller-owned retrieves, pass unparsed provider counts through, log batch usage in chat shape

This commit is contained in:
mateo-berri 2026-10-29 10:15:46 -07:00
parent 7c36b6ed20
commit 2a852fa95b
4 changed files with 92 additions and 16 deletions

View file

@ -20,6 +20,7 @@ from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from httpx import Response, Timeout
from openai.types.batch_usage import BatchUsage
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -2071,7 +2072,9 @@ class Logging(LiteLLMLoggingBaseClass):
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
router_model_id = hidden_params["model_id"]
if isinstance(priced_result, LiteLLMBatch) and not batch_cost_is_final(priced_result):
if isinstance(priced_result, LiteLLMBatch) and (
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
):
return 0.0
# Fallback: extract router_model_id from litellm_params when not available
@ -3278,8 +3281,6 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs: Any, # kwargs-ok: forwarded from success_handler
) -> None:
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if self._is_batch_poll_without_logging(result):
return
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
@ -7252,6 +7253,15 @@ def _served_from_cache_at_no_charge(result: object) -> bool:
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields)
def _batch_chat_usage(response: object) -> Usage | None:
if not isinstance(response, LiteLLMBatch) or response.usage is None:
return None
return Usage.model_validate(response.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS)))
def _extract_response_obj_and_hidden_params(
init_response_obj: object,
original_exception: Exception | None,
@ -7389,7 +7399,8 @@ def get_standard_logging_object_payload(
# Extract usage as a plain dict, avoiding Pydantic round-trip
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
or _batch_chat_usage(init_response_obj),
)
usage_dict: Final = (
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}

View file

@ -4845,6 +4845,12 @@ def _provider_batch_usage_details(usage: Usage) -> _ProviderBatchUsageDetails:
return _ProviderBatchUsageDetails()
def _has_unparsed_batch_token_counts(usage: Usage) -> bool:
provider_fields: Final = usage.model_extra or {}
sent_batch_counts: Final = "input_tokens" in provider_fields or "output_tokens" in provider_fields
return sent_batch_counts and _parse_batch_token_counts(provider_fields) is None
def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
provider_details: Final = _provider_batch_usage_details(usage)
cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None
@ -4889,15 +4895,18 @@ class LiteLLMBatch(Batch):
) -> Mapping[str, object] | None:
if not isinstance(usage, Usage):
return usage
chat_fields: Final = usage.model_dump(
mode=info.mode,
include=info.include,
exclude=info.exclude,
exclude_unset=info.exclude_unset,
exclude_defaults=info.exclude_defaults,
exclude_none=info.exclude_none,
)
if _has_unparsed_batch_token_counts(usage):
return chat_fields
return {
**usage.model_dump(
mode=info.mode,
include=info.include,
exclude=info.exclude,
exclude_unset=info.exclude_unset,
exclude_defaults=info.exclude_defaults,
exclude_none=info.exclude_none,
),
**chat_fields,
**_batch_usage_from_chat_usage(usage).model_dump(
mode=info.mode,
include=info.include,

View file

@ -1591,12 +1591,18 @@ class TestRetrieveBatchPricesOnlyFinalBatches:
assert batch.usage is not None
assert batch.usage.total_tokens == 35
@pytest.mark.parametrize(("batch_ignore_default_logging", "expected_events"), [(True, 0), (False, 1)])
def test_sync_logger_skips_the_cost_poller_retrieve(self, batch_ignore_default_logging, expected_events) -> None:
@pytest.mark.parametrize("batch_ignore_default_logging", [True, False])
def test_sync_logger_gets_a_free_event_for_the_cost_poller_retrieve(self, batch_ignore_default_logging: bool) -> None:
sync_events: list[object] = []
class SyncOnlyLogger(CustomLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
def log_success_event(
self,
kwargs: dict[str, object],
response_obj: object,
start_time: dt_object,
end_time: dt_object,
) -> None:
sync_events.append(kwargs["response_cost"])
logging_obj = LitellmLogging(
@ -1624,7 +1630,40 @@ class TestRetrieveBatchPricesOnlyFinalBatches:
for_async_request=True,
)
assert len(sync_events) == expected_events
assert len(sync_events) == 1
assert isinstance(sync_events[0], float)
assert (sync_events[0] == 0.0) is batch_ignore_default_logging
def test_logged_usage_object_keeps_the_batch_chat_usage(self) -> None:
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
from litellm.types.utils import Usage
aggregate: Final = Usage(
prompt_tokens=29, completion_tokens=48, total_tokens=77, reasoning_tokens=3, cache_creation_input_tokens=300
)
billed: Final = self._batch("completed", "file-out").model_copy(update={"usage": aggregate})
provider_read: Final = self._batch("completed", "file-out")
now: Final = dt_object(2026, 1, 1, 12, 0, 0)
usage_objects: Final = tuple(
payload["metadata"]["usage_object"]
for payload in (
get_standard_logging_object_payload(
kwargs={"litellm_params": {}},
init_response_obj=batch,
start_time=now,
end_time=now,
logging_obj=self._logging_obj(),
status="success",
)
for batch in (billed, provider_read)
)
if payload is not None
)
assert usage_objects[0] == aggregate.model_dump()
assert usage_objects[1]["prompt_tokens"] == 29
assert "input_tokens" not in usage_objects[1]
class TestAnthropicPassthroughCustomPricing:

View file

@ -1189,6 +1189,23 @@ def test_batch_dump_forwards_every_provider_usage_detail_field() -> None:
assert python_usage["output_tokens_details"] == {"reasoning_tokens": 448, "audio_tokens": 2}
@pytest.mark.parametrize(
"usage",
[
{"input_tokens": 29, "output_tokens": None, "total_tokens": 29},
{"input_tokens": 29, "input_tokens_details": {"cached_tokens": 8}, "total_tokens": 29},
],
)
def test_batch_dump_serves_unparsed_provider_usage_unchanged(usage: dict[str, object]) -> None:
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
python_usage: Final = batch.model_dump()["usage"]
json_usage: Final = json.loads(batch.model_dump_json(warnings=False))["usage"]
assert python_usage == json_usage
assert {key: python_usage[key] for key in usage} == usage
def test_batch_dump_applies_exclude_none_to_provider_usage_detail_fields() -> None:
batch: Final = LiteLLMBatch.model_validate(
{