From 64eb016b3d149e17852b615dc27fa91b32adde2d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:52:22 -0700 Subject: [PATCH 1/4] fix(batches): read OpenAI batch usage into the batch token counts --- litellm/types/utils.py | 33 +++++++++++++++++ tests/unit/types/test_types_utils.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..2a4c04e3c01 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -20,6 +20,7 @@ from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import ( FileTypes as FileTypes, ) +from openai.types.batch_usage import BatchUsage from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion from openai.types.completion_usage import ( CompletionTokensDetails, @@ -44,6 +45,7 @@ from pydantic import ( JsonValue, PrivateAttr, SkipValidation, + ValidationError, field_serializer, field_validator, ) @@ -4720,6 +4722,31 @@ class LiteLLMFineTuningJob(FineTuningJob): self._hidden_params = kwargs.get("_hidden_params", {}) +def _parse_batch_usage(value: object) -> BatchUsage | None: + if isinstance(value, BatchUsage): + return value + if not isinstance(value, Mapping): + return None + try: + return BatchUsage.model_validate(value) + except ValidationError: + return None + + +def _chat_usage_from_batch_usage(batch_usage: BatchUsage) -> Usage: + return Usage( + prompt_tokens=batch_usage.input_tokens, + completion_tokens=batch_usage.output_tokens, + total_tokens=batch_usage.total_tokens, + prompt_tokens_details={"cached_tokens": batch_usage.input_tokens_details.cached_tokens}, + completion_tokens_details={"reasoning_tokens": batch_usage.output_tokens_details.reasoning_tokens}, + input_tokens=batch_usage.input_tokens, + input_tokens_details=batch_usage.input_tokens_details.model_dump(), + output_tokens=batch_usage.output_tokens, + output_tokens_details=batch_usage.output_tokens_details.model_dump(), + ) + + class LiteLLMBatch(Batch): _hidden_params: dict = {} @@ -4733,6 +4760,12 @@ class LiteLLMBatch(Batch): usage: Usage | None = None + @field_validator("usage", mode="before") + @classmethod + def _usage_from_batch_usage(cls, value: object) -> object: + batch_usage: Final = _parse_batch_usage(value) + return value if batch_usage is None else _chat_usage_from_batch_usage(batch_usage) + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index d85e48540f9..9095f0721f4 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -25,6 +25,7 @@ from litellm.types.utils import ( HiddenParams, ImageObject, ImageResponse, + LiteLLMBatch, ModelResponse, ModelResponseStream, TextCompletionResponse, @@ -1007,3 +1008,56 @@ def test_text_tokens_without_nested_reasoning_clamps( ) == expected_text_tokens ) + + +_BATCH_FIELDS: Final = { + "id": "batch_abc", + "object": "batch", + "endpoint": "/v1/responses", + "input_file_id": "file-abc", + "completion_window": "24h", + "status": "completed", + "created_at": 1, +} +_PROVIDER_BATCH_USAGE: Final = { + "input_tokens": 21, + "input_tokens_details": {"cached_tokens": 5}, + "output_tokens": 543, + "output_tokens_details": {"reasoning_tokens": 448}, + "total_tokens": 564, +} + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"), + pytest.param( + {"prompt_tokens": 0, "completion_tokens": 0, **_PROVIDER_BATCH_USAGE}, + id="stored_row_with_zeroed_chat_fields", + ), + ), +) +def test_batch_usage_reads_token_counts_from_the_provider_batch_usage(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + + assert batch.usage is not None + assert batch.usage.prompt_tokens_details is not None + assert batch.usage.completion_tokens_details is not None + assert ( + batch.usage.prompt_tokens, + batch.usage.completion_tokens, + batch.usage.total_tokens, + batch.usage.prompt_tokens_details.cached_tokens, + batch.usage.completion_tokens_details.reasoning_tokens, + ) == (21, 543, 564, 5, 448) + assert (batch.usage.input_tokens, batch.usage.output_tokens) == (21, 543) + + +def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None: + batch: Final = LiteLLMBatch.model_validate( + {**_BATCH_FIELDS, "usage": {"prompt_tokens": 28, "completion_tokens": 8, "total_tokens": 36}} + ) + + assert batch.usage is not None + assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36) From 7f98570c925b974c1657d761d11f86af4c010e0a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:03:35 -0700 Subject: [PATCH 2/4] fix(batches): serve OpenAI's batch usage shape from retrieve and the cost poller --- litellm/types/utils.py | 20 ++++++- .../proxy/test_managed_files_access_check.py | 4 +- .../common_utils/test_check_batch_cost.py | 13 +++-- tests/unit/types/test_types_utils.py | 54 +++++++++++++++---- 4 files changed, 76 insertions(+), 15 deletions(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2a4c04e3c01..c4254b11376 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -20,7 +20,7 @@ from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import ( FileTypes as FileTypes, ) -from openai.types.batch_usage import BatchUsage +from openai.types.batch_usage import BatchUsage, InputTokensDetails, OutputTokensDetails from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion from openai.types.completion_usage import ( CompletionTokensDetails, @@ -4747,6 +4747,20 @@ def _chat_usage_from_batch_usage(batch_usage: BatchUsage) -> Usage: ) +def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage: + cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None + reasoning_tokens: Final = ( + usage.completion_tokens_details.reasoning_tokens if usage.completion_tokens_details else None + ) + return BatchUsage( + input_tokens=usage.prompt_tokens, + input_tokens_details=InputTokensDetails(cached_tokens=cached_tokens or 0), + output_tokens=usage.completion_tokens, + output_tokens_details=OutputTokensDetails(reasoning_tokens=reasoning_tokens or 0), + total_tokens=usage.total_tokens, + ) + + class LiteLLMBatch(Batch): _hidden_params: dict = {} @@ -4766,6 +4780,10 @@ class LiteLLMBatch(Batch): batch_usage: Final = _parse_batch_usage(value) return value if batch_usage is None else _chat_usage_from_batch_usage(batch_usage) + @field_serializer("usage", when_used="json") + def _serialize_usage_as_batch_usage(self, usage: Usage | None) -> BatchUsage | None: + return _batch_usage_from_chat_usage(usage) if isinstance(usage, Usage) else usage + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) diff --git a/tests/unit/enterprise/proxy/test_managed_files_access_check.py b/tests/unit/enterprise/proxy/test_managed_files_access_check.py index 0cdc212502c..0c51dfa5ec9 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_access_check.py +++ b/tests/unit/enterprise/proxy/test_managed_files_access_check.py @@ -378,5 +378,5 @@ async def test_check_batch_cost_should_call_afile_content_directly_with_credenti assert claim_data["status"] == "complete" persisted_batch: Final = json.loads(claim_data["file_object"]) assert persisted_batch["status"] == "completed" - assert persisted_batch["usage"]["prompt_tokens"] == 10 - assert persisted_batch["usage"]["completion_tokens"] == 5 + assert persisted_batch["usage"]["input_tokens"] == 10 + assert persisted_batch["usage"]["output_tokens"] == 5 diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 33b8085e082..d529e09f5be 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -70,10 +70,11 @@ def _batch_cost_result( """Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns, for mocking it in tests that only care about cost/usage/models.""" from litellm.batches.batch_utils import BatchCostUsageResult + from litellm.types.utils import Usage return BatchCostUsageResult( cost=cost, - usage=usage, + usage=Usage(**usage), models=models, successful_requests=successful_requests, failed_requests=failed_requests, @@ -3094,7 +3095,7 @@ class TestMultiPodBatchCostClaim: patch( "litellm.batches.batch_utils.calculate_batch_cost_and_usage", new_callable=AsyncMock, - return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]), + return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]), ), patch( "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", @@ -3509,7 +3510,13 @@ class TestMultiPodBatchCostClaim: assert journal == ["fetch", "claim", "bill"] assert row.status == "complete" - assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5} + assert json.loads(row.file_object)["usage"] == { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 15, + } logging_obj.async_success_handler.assert_awaited_once() assert row.batch_processed is True diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index 9095f0721f4..afdb3ddbc3a 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -31,6 +31,7 @@ from litellm.types.utils import ( TextCompletionResponse, all_litellm_params, is_decisions_model_mode, + Usage, text_tokens_without_nested_reasoning, ) @@ -1028,6 +1029,19 @@ _PROVIDER_BATCH_USAGE: Final = { } +def _batch_token_counts(batch: LiteLLMBatch) -> tuple[int, int, int, int | None, int | None]: + assert batch.usage is not None + assert batch.usage.prompt_tokens_details is not None + assert batch.usage.completion_tokens_details is not None + return ( + batch.usage.prompt_tokens, + batch.usage.completion_tokens, + batch.usage.total_tokens, + batch.usage.prompt_tokens_details.cached_tokens, + batch.usage.completion_tokens_details.reasoning_tokens, + ) + + @pytest.mark.parametrize( "usage", ( @@ -1041,16 +1055,8 @@ _PROVIDER_BATCH_USAGE: Final = { def test_batch_usage_reads_token_counts_from_the_provider_batch_usage(usage: dict[str, object]) -> None: batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + assert _batch_token_counts(batch) == (21, 543, 564, 5, 448) assert batch.usage is not None - assert batch.usage.prompt_tokens_details is not None - assert batch.usage.completion_tokens_details is not None - assert ( - batch.usage.prompt_tokens, - batch.usage.completion_tokens, - batch.usage.total_tokens, - batch.usage.prompt_tokens_details.cached_tokens, - batch.usage.completion_tokens_details.reasoning_tokens, - ) == (21, 543, 564, 5, 448) assert (batch.usage.input_tokens, batch.usage.output_tokens) == (21, 543) @@ -1061,3 +1067,33 @@ def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None: assert batch.usage is not None assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"), + pytest.param( + Usage( + prompt_tokens=21, + completion_tokens=543, + total_tokens=564, + prompt_tokens_details={"cached_tokens": 5}, + completion_tokens_details={"reasoning_tokens": 448}, + ), + id="billed_chat_usage", + ), + ), +) +def test_batch_json_carries_the_provider_batch_usage_shape(usage: dict[str, object] | Usage) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + dumped: Final = batch.model_dump_json() + + assert json.loads(dumped)["usage"] == _PROVIDER_BATCH_USAGE + assert _batch_token_counts(LiteLLMBatch.model_validate_json(dumped)) == (21, 543, 564, 5, 448) + + +def test_batch_json_keeps_usage_that_skipped_validation() -> None: + batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(update={"usage": {"prompt_tokens": 3}}) + + assert json.loads(batch.model_dump_json(warnings=False))["usage"] == {"prompt_tokens": 3} From 7c7371c288361164d45c4130efb5660a83b66a69 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:57:28 -0700 Subject: [PATCH 3/4] fix(batches): keep the chat usage keys next to OpenAI's batch usage fields Batch usage now carries every chat key it had plus OpenAI's five batch fields with the same counts, so poller-billed batches keep prompt_tokens and cache-write tokens, and older proxies still read the new rows. Usage with missing or null detail counts reads its real input and output counts. --- litellm/types/utils.py | 91 ++++++++++++++----- .../proxy/test_managed_files_access_check.py | 4 +- .../common_utils/test_check_batch_cost.py | 6 +- tests/unit/types/test_types_utils.py | 50 +++++++++- 4 files changed, 125 insertions(+), 26 deletions(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c4254b11376..a3e82bdf785 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4722,28 +4722,70 @@ class LiteLLMFineTuningJob(FineTuningJob): self._hidden_params = kwargs.get("_hidden_params", {}) -def _parse_batch_usage(value: object) -> BatchUsage | None: - if isinstance(value, BatchUsage): - return value +class _BatchInputTokensDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + cached_tokens: int | None = None + + +class _BatchOutputTokensDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + reasoning_tokens: int | None = None + + +class _BatchTokenCounts(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + output_tokens: int + total_tokens: int | None = None + input_tokens_details: _BatchInputTokensDetails | None = None + output_tokens_details: _BatchOutputTokensDetails | None = None + + +def _parse_batch_token_counts(value: object) -> _BatchTokenCounts | None: if not isinstance(value, Mapping): return None try: - return BatchUsage.model_validate(value) + return _BatchTokenCounts.model_validate(value) except ValidationError: return None -def _chat_usage_from_batch_usage(batch_usage: BatchUsage) -> Usage: - return Usage( - prompt_tokens=batch_usage.input_tokens, - completion_tokens=batch_usage.output_tokens, - total_tokens=batch_usage.total_tokens, - prompt_tokens_details={"cached_tokens": batch_usage.input_tokens_details.cached_tokens}, - completion_tokens_details={"reasoning_tokens": batch_usage.output_tokens_details.reasoning_tokens}, - input_tokens=batch_usage.input_tokens, - input_tokens_details=batch_usage.input_tokens_details.model_dump(), - output_tokens=batch_usage.output_tokens, - output_tokens_details=batch_usage.output_tokens_details.model_dump(), +def _prompt_details_with_cached_tokens( + details: PromptTokensDetailsWrapper | None, cached_tokens: int | None +) -> PromptTokensDetailsWrapper | None: + if not cached_tokens: + return details + return (details or PromptTokensDetailsWrapper()).model_copy(update={"cached_tokens": cached_tokens}) + + +def _completion_details_with_reasoning_tokens( + details: CompletionTokensDetailsWrapper | None, reasoning_tokens: int | None +) -> CompletionTokensDetailsWrapper | None: + if not reasoning_tokens: + return details + return (details or CompletionTokensDetailsWrapper()).model_copy(update={"reasoning_tokens": reasoning_tokens}) + + +def _usage_with_batch_token_counts(usage: Usage, counts: _BatchTokenCounts) -> Usage: + return usage.model_copy( + update={ + "prompt_tokens": counts.input_tokens, + "completion_tokens": counts.output_tokens, + "total_tokens": ( + counts.input_tokens + counts.output_tokens if counts.total_tokens is None else counts.total_tokens + ), + "prompt_tokens_details": _prompt_details_with_cached_tokens( + usage.prompt_tokens_details, + counts.input_tokens_details.cached_tokens if counts.input_tokens_details else None, + ), + "completion_tokens_details": _completion_details_with_reasoning_tokens( + usage.completion_tokens_details, + counts.output_tokens_details.reasoning_tokens if counts.output_tokens_details else None, + ), + } ) @@ -4776,13 +4818,20 @@ class LiteLLMBatch(Batch): @field_validator("usage", mode="before") @classmethod - def _usage_from_batch_usage(cls, value: object) -> object: - batch_usage: Final = _parse_batch_usage(value) - return value if batch_usage is None else _chat_usage_from_batch_usage(batch_usage) + def _usage_from_batch_token_counts(cls, value: object) -> object: + counts: Final = _parse_batch_token_counts(value) + return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts) - @field_serializer("usage", when_used="json") - def _serialize_usage_as_batch_usage(self, usage: Usage | None) -> BatchUsage | None: - return _batch_usage_from_chat_usage(usage) if isinstance(usage, Usage) else usage + @field_serializer("usage") + def _serialize_usage_with_batch_fields( + self, usage: Usage | None, info: FieldSerializationInfo + ) -> Mapping[str, object] | None: + if not isinstance(usage, Usage): + return usage + return { + **usage.model_dump(mode=info.mode, exclude_none=info.exclude_none), + **_batch_usage_from_chat_usage(usage).model_dump(mode=info.mode), + } def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/unit/enterprise/proxy/test_managed_files_access_check.py b/tests/unit/enterprise/proxy/test_managed_files_access_check.py index 0c51dfa5ec9..0cdc212502c 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_access_check.py +++ b/tests/unit/enterprise/proxy/test_managed_files_access_check.py @@ -378,5 +378,5 @@ async def test_check_batch_cost_should_call_afile_content_directly_with_credenti assert claim_data["status"] == "complete" persisted_batch: Final = json.loads(claim_data["file_object"]) assert persisted_batch["status"] == "completed" - assert persisted_batch["usage"]["input_tokens"] == 10 - assert persisted_batch["usage"]["output_tokens"] == 5 + assert persisted_batch["usage"]["prompt_tokens"] == 10 + assert persisted_batch["usage"]["completion_tokens"] == 5 diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index d529e09f5be..40c244f67fe 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -3511,11 +3511,15 @@ class TestMultiPodBatchCostClaim: assert journal == ["fetch", "claim", "bill"] assert row.status == "complete" assert json.loads(row.file_object)["usage"] == { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": None, + "completion_tokens_details": None, "input_tokens": 10, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, - "total_tokens": 15, } logging_obj.async_success_handler.assert_awaited_once() assert row.batch_processed is True diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index afdb3ddbc3a..05aef9a7c51 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -1069,6 +1069,35 @@ def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None: assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36) +@pytest.mark.parametrize( + "usage", + ( + pytest.param({"input_tokens": 10, "output_tokens": 5}, id="no_details_or_total"), + pytest.param( + {"input_tokens": 10, "input_tokens_details": None, "output_tokens": 5, "output_tokens_details": None}, + id="null_details", + ), + pytest.param( + { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": None}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": None}, + "total_tokens": 15, + }, + id="null_detail_counts", + ), + ), +) +def test_batch_usage_reads_token_counts_from_partial_provider_batch_usage(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + dumped_usage: Final = json.loads(batch.model_dump_json())["usage"] + + assert batch.usage is not None + assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (10, 5, 15) + assert (dumped_usage["input_tokens"], dumped_usage["output_tokens"], dumped_usage["total_tokens"]) == (10, 5, 15) + + @pytest.mark.parametrize( "usage", ( @@ -1085,14 +1114,31 @@ def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None: ), ), ) -def test_batch_json_carries_the_provider_batch_usage_shape(usage: dict[str, object] | Usage) -> None: +def test_batch_dump_carries_the_chat_and_provider_batch_usage_fields(usage: dict[str, object] | Usage) -> None: batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) dumped: Final = batch.model_dump_json() + dumped_usage: Final = json.loads(dumped)["usage"] - assert json.loads(dumped)["usage"] == _PROVIDER_BATCH_USAGE + assert {key: dumped_usage[key] for key in _PROVIDER_BATCH_USAGE} == _PROVIDER_BATCH_USAGE + assert (dumped_usage["prompt_tokens"], dumped_usage["completion_tokens"]) == (21, 543) + assert batch.model_dump()["usage"] == dumped_usage assert _batch_token_counts(LiteLLMBatch.model_validate_json(dumped)) == (21, 543, 564, 5, 448) +def test_batch_dump_keeps_cache_write_tokens_from_billed_usage() -> None: + batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy( + update={"usage": Usage(prompt_tokens=30, completion_tokens=4, total_tokens=34, cache_creation_input_tokens=7)} + ) + dumped: Final = batch.model_dump_json() + dumped_usage: Final = json.loads(dumped)["usage"] + + assert (dumped_usage["prompt_tokens"], dumped_usage["input_tokens"]) == (30, 30) + assert (dumped_usage["completion_tokens"], dumped_usage["output_tokens"]) == (4, 4) + assert dumped_usage["prompt_tokens_details"]["cache_write_tokens"] == 7 + assert None not in batch.model_dump(exclude_none=True)["usage"]["prompt_tokens_details"].values() + assert json.loads(LiteLLMBatch.model_validate_json(dumped).model_dump_json())["usage"] == dumped_usage + + def test_batch_json_keeps_usage_that_skipped_validation() -> None: batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(update={"usage": {"prompt_tokens": 3}}) From 59608b378585ded2fd626d8f7b92870358ab7b99 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:39:50 -0700 Subject: [PATCH 4/4] test(batches): check the batch usage counts in the e2e terminal-state test --- tests/e2e/batches/batch_client.py | 13 ++++++++++++- tests/e2e/batches/test_batches_e2e.py | 7 +++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index b02b09e557b..598593c1fcd 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -15,7 +15,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Final, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from e2e_metadata import step from proxy_client import ProxyClient @@ -77,6 +77,16 @@ class BatchErrorList(BaseModel): data: list[BatchErrorItem] = [] +class BatchUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None + input_tokens: int | None = None + output_tokens: int | None = None + + class BatchObject(BaseModel): id: str object: str | None = None @@ -90,6 +100,7 @@ class BatchObject(BaseModel): completion_window: str | None = None created_at: int | None = None model: str | None = None + usage: BatchUsage | None = None class BatchList(BaseModel): diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 54bcb80ed24..b7a52da5902 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1893,6 +1893,13 @@ class TestBatchTerminalState: f"listed-completed marker retrieved as {fetched.status!r}" ) assert fetched.output_file_id, "completed batch has no output_file_id" + usage: Final = fetched.usage + assert usage is not None and usage.input_tokens and usage.output_tokens, ( + f"completed batch has no batch token counts: {usage!r}" + ) + assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), ( + f"chat token counts disagree with the batch token counts: {usage!r}" + ) downloaded = client.proxy.transport.download( f"/v1/files/{fetched.output_file_id}/content",