mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(batches): keep sync logger events at $0 on poller-owned retrieves, pass unparsed provider counts through, log batch usage in chat shape
This commit is contained in:
parent
7c36b6ed20
commit
2a852fa95b
4 changed files with 92 additions and 16 deletions
|
|
@ -20,6 +20,7 @@ from types import MappingProxyType, TracebackType
|
|||
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
|
||||
|
||||
from httpx import Response, Timeout
|
||||
from openai.types.batch_usage import BatchUsage
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
|
|
@ -2071,7 +2072,9 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
|
||||
router_model_id = hidden_params["model_id"]
|
||||
|
||||
if isinstance(priced_result, LiteLLMBatch) and not batch_cost_is_final(priced_result):
|
||||
if isinstance(priced_result, LiteLLMBatch) and (
|
||||
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
|
||||
):
|
||||
return 0.0
|
||||
|
||||
# Fallback: extract router_model_id from litellm_params when not available
|
||||
|
|
@ -3278,8 +3281,6 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
**kwargs: Any, # kwargs-ok: forwarded from success_handler
|
||||
) -> None:
|
||||
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
|
||||
if self._is_batch_poll_without_logging(result):
|
||||
return
|
||||
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
|
||||
return
|
||||
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
|
||||
|
|
@ -7252,6 +7253,15 @@ def _served_from_cache_at_no_charge(result: object) -> bool:
|
|||
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
|
||||
|
||||
|
||||
_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields)
|
||||
|
||||
|
||||
def _batch_chat_usage(response: object) -> Usage | None:
|
||||
if not isinstance(response, LiteLLMBatch) or response.usage is None:
|
||||
return None
|
||||
return Usage.model_validate(response.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS)))
|
||||
|
||||
|
||||
def _extract_response_obj_and_hidden_params(
|
||||
init_response_obj: object,
|
||||
original_exception: Exception | None,
|
||||
|
|
@ -7389,7 +7399,8 @@ def get_standard_logging_object_payload(
|
|||
# Extract usage as a plain dict, avoiding Pydantic round-trip
|
||||
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
|
||||
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
|
||||
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
|
||||
or _batch_chat_usage(init_response_obj),
|
||||
)
|
||||
usage_dict: Final = (
|
||||
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
|
||||
|
|
|
|||
|
|
@ -4845,6 +4845,12 @@ def _provider_batch_usage_details(usage: Usage) -> _ProviderBatchUsageDetails:
|
|||
return _ProviderBatchUsageDetails()
|
||||
|
||||
|
||||
def _has_unparsed_batch_token_counts(usage: Usage) -> bool:
|
||||
provider_fields: Final = usage.model_extra or {}
|
||||
sent_batch_counts: Final = "input_tokens" in provider_fields or "output_tokens" in provider_fields
|
||||
return sent_batch_counts and _parse_batch_token_counts(provider_fields) is None
|
||||
|
||||
|
||||
def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
|
||||
provider_details: Final = _provider_batch_usage_details(usage)
|
||||
cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None
|
||||
|
|
@ -4889,15 +4895,18 @@ class LiteLLMBatch(Batch):
|
|||
) -> Mapping[str, object] | None:
|
||||
if not isinstance(usage, Usage):
|
||||
return usage
|
||||
chat_fields: Final = usage.model_dump(
|
||||
mode=info.mode,
|
||||
include=info.include,
|
||||
exclude=info.exclude,
|
||||
exclude_unset=info.exclude_unset,
|
||||
exclude_defaults=info.exclude_defaults,
|
||||
exclude_none=info.exclude_none,
|
||||
)
|
||||
if _has_unparsed_batch_token_counts(usage):
|
||||
return chat_fields
|
||||
return {
|
||||
**usage.model_dump(
|
||||
mode=info.mode,
|
||||
include=info.include,
|
||||
exclude=info.exclude,
|
||||
exclude_unset=info.exclude_unset,
|
||||
exclude_defaults=info.exclude_defaults,
|
||||
exclude_none=info.exclude_none,
|
||||
),
|
||||
**chat_fields,
|
||||
**_batch_usage_from_chat_usage(usage).model_dump(
|
||||
mode=info.mode,
|
||||
include=info.include,
|
||||
|
|
|
|||
|
|
@ -1591,12 +1591,18 @@ class TestRetrieveBatchPricesOnlyFinalBatches:
|
|||
assert batch.usage is not None
|
||||
assert batch.usage.total_tokens == 35
|
||||
|
||||
@pytest.mark.parametrize(("batch_ignore_default_logging", "expected_events"), [(True, 0), (False, 1)])
|
||||
def test_sync_logger_skips_the_cost_poller_retrieve(self, batch_ignore_default_logging, expected_events) -> None:
|
||||
@pytest.mark.parametrize("batch_ignore_default_logging", [True, False])
|
||||
def test_sync_logger_gets_a_free_event_for_the_cost_poller_retrieve(self, batch_ignore_default_logging: bool) -> None:
|
||||
sync_events: list[object] = []
|
||||
|
||||
class SyncOnlyLogger(CustomLogger):
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
def log_success_event(
|
||||
self,
|
||||
kwargs: dict[str, object],
|
||||
response_obj: object,
|
||||
start_time: dt_object,
|
||||
end_time: dt_object,
|
||||
) -> None:
|
||||
sync_events.append(kwargs["response_cost"])
|
||||
|
||||
logging_obj = LitellmLogging(
|
||||
|
|
@ -1624,7 +1630,40 @@ class TestRetrieveBatchPricesOnlyFinalBatches:
|
|||
for_async_request=True,
|
||||
)
|
||||
|
||||
assert len(sync_events) == expected_events
|
||||
assert len(sync_events) == 1
|
||||
assert isinstance(sync_events[0], float)
|
||||
assert (sync_events[0] == 0.0) is batch_ignore_default_logging
|
||||
|
||||
def test_logged_usage_object_keeps_the_batch_chat_usage(self) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
aggregate: Final = Usage(
|
||||
prompt_tokens=29, completion_tokens=48, total_tokens=77, reasoning_tokens=3, cache_creation_input_tokens=300
|
||||
)
|
||||
billed: Final = self._batch("completed", "file-out").model_copy(update={"usage": aggregate})
|
||||
provider_read: Final = self._batch("completed", "file-out")
|
||||
now: Final = dt_object(2026, 1, 1, 12, 0, 0)
|
||||
|
||||
usage_objects: Final = tuple(
|
||||
payload["metadata"]["usage_object"]
|
||||
for payload in (
|
||||
get_standard_logging_object_payload(
|
||||
kwargs={"litellm_params": {}},
|
||||
init_response_obj=batch,
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
logging_obj=self._logging_obj(),
|
||||
status="success",
|
||||
)
|
||||
for batch in (billed, provider_read)
|
||||
)
|
||||
if payload is not None
|
||||
)
|
||||
|
||||
assert usage_objects[0] == aggregate.model_dump()
|
||||
assert usage_objects[1]["prompt_tokens"] == 29
|
||||
assert "input_tokens" not in usage_objects[1]
|
||||
|
||||
|
||||
class TestAnthropicPassthroughCustomPricing:
|
||||
|
|
|
|||
|
|
@ -1189,6 +1189,23 @@ def test_batch_dump_forwards_every_provider_usage_detail_field() -> None:
|
|||
assert python_usage["output_tokens_details"] == {"reasoning_tokens": 448, "audio_tokens": 2}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"usage",
|
||||
[
|
||||
{"input_tokens": 29, "output_tokens": None, "total_tokens": 29},
|
||||
{"input_tokens": 29, "input_tokens_details": {"cached_tokens": 8}, "total_tokens": 29},
|
||||
],
|
||||
)
|
||||
def test_batch_dump_serves_unparsed_provider_usage_unchanged(usage: dict[str, object]) -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
|
||||
|
||||
python_usage: Final = batch.model_dump()["usage"]
|
||||
json_usage: Final = json.loads(batch.model_dump_json(warnings=False))["usage"]
|
||||
|
||||
assert python_usage == json_usage
|
||||
assert {key: python_usage[key] for key in usage} == usage
|
||||
|
||||
|
||||
def test_batch_dump_applies_exclude_none_to_provider_usage_detail_fields() -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate(
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue