This commit is contained in:
Mateo Wang 2026-10-28 08:27:05 -07:00 • committed by GitHub
commit 9e062832e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 269 additions and 4 deletions

View file

@ -20,6 +20,7 @@ from openai._models import BaseModel as OpenAIObject
from openai.types.audio.transcription_create_params import (
FileTypes as FileTypes,
)
from openai.types.batch_usage import BatchUsage, InputTokensDetails, OutputTokensDetails
from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion
from openai.types.completion_usage import (
CompletionTokensDetails,
@ -44,6 +45,7 @@ from pydantic import (
JsonValue,
PrivateAttr,
SkipValidation,
ValidationError,
field_serializer,
field_validator,
)
@ -4758,6 +4760,87 @@ class LiteLLMFineTuningJob(FineTuningJob):
self._hidden_params = kwargs.get("_hidden_params", {})
class _BatchInputTokensDetails(BaseModel):
model_config = ConfigDict(frozen=True)
cached_tokens: int | None = None
class _BatchOutputTokensDetails(BaseModel):
model_config = ConfigDict(frozen=True)
reasoning_tokens: int | None = None
class _BatchTokenCounts(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: int
output_tokens: int
total_tokens: int | None = None
input_tokens_details: _BatchInputTokensDetails | None = None
output_tokens_details: _BatchOutputTokensDetails | None = None
def _parse_batch_token_counts(value: object) -> _BatchTokenCounts | None:
if not isinstance(value, Mapping):
return None
try:
return _BatchTokenCounts.model_validate(value)
except ValidationError:
return None
def _prompt_details_with_cached_tokens(
details: PromptTokensDetailsWrapper | None, cached_tokens: int | None
) -> PromptTokensDetailsWrapper | None:
if not cached_tokens:
return details
return (details or PromptTokensDetailsWrapper()).model_copy(update={"cached_tokens": cached_tokens})
def _completion_details_with_reasoning_tokens(
details: CompletionTokensDetailsWrapper | None, reasoning_tokens: int | None
) -> CompletionTokensDetailsWrapper | None:
if not reasoning_tokens:
return details
return (details or CompletionTokensDetailsWrapper()).model_copy(update={"reasoning_tokens": reasoning_tokens})
def _usage_with_batch_token_counts(usage: Usage, counts: _BatchTokenCounts) -> Usage:
return usage.model_copy(
update={
"prompt_tokens": counts.input_tokens,
"completion_tokens": counts.output_tokens,
"total_tokens": (
counts.input_tokens + counts.output_tokens if counts.total_tokens is None else counts.total_tokens
),
"prompt_tokens_details": _prompt_details_with_cached_tokens(
usage.prompt_tokens_details,
counts.input_tokens_details.cached_tokens if counts.input_tokens_details else None,
),
"completion_tokens_details": _completion_details_with_reasoning_tokens(
usage.completion_tokens_details,
counts.output_tokens_details.reasoning_tokens if counts.output_tokens_details else None,
),
}
)
def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None
reasoning_tokens: Final = (
usage.completion_tokens_details.reasoning_tokens if usage.completion_tokens_details else None
)
return BatchUsage(
input_tokens=usage.prompt_tokens,
input_tokens_details=InputTokensDetails(cached_tokens=cached_tokens or 0),
output_tokens=usage.completion_tokens,
output_tokens_details=OutputTokensDetails(reasoning_tokens=reasoning_tokens or 0),
total_tokens=usage.total_tokens,
)
class LiteLLMBatch(Batch):
_hidden_params: dict = {}
@ -4771,6 +4854,23 @@ class LiteLLMBatch(Batch):
usage: Usage | None = None
@field_validator("usage", mode="before")
@classmethod
def _usage_from_batch_token_counts(cls, value: object) -> object:
counts: Final = _parse_batch_token_counts(value)
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
@field_serializer("usage")
def _serialize_usage_with_batch_fields(
self, usage: Usage | None, info: FieldSerializationInfo
) -> Mapping[str, object] | None:
if not isinstance(usage, Usage):
return usage
return {
**usage.model_dump(mode=info.mode, exclude_none=info.exclude_none),
**_batch_usage_from_chat_usage(usage).model_dump(mode=info.mode),
}
def __contains__(self, key) -> bool:
# Define custom behavior for the 'in' operator
return hasattr(self, key)

View file

@ -15,7 +15,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Final, Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field
from e2e_metadata import step
from proxy_client import ProxyClient
@ -77,6 +77,16 @@ class BatchErrorList(BaseModel):
data: list[BatchErrorItem] = []
class BatchUsage(BaseModel):
model_config = ConfigDict(frozen=True)
prompt_tokens: int | None = None
completion_tokens: int | None = None
total_tokens: int | None = None
input_tokens: int | None = None
output_tokens: int | None = None
class BatchObject(BaseModel):
id: str
object: str | None = None
@ -90,6 +100,7 @@ class BatchObject(BaseModel):
completion_window: str | None = None
created_at: int | None = None
model: str | None = None
usage: BatchUsage | None = None
class BatchList(BaseModel):

View file

@ -1893,6 +1893,13 @@ class TestBatchTerminalState:
f"listed-completed marker retrieved as {fetched.status!r}"
)
assert fetched.output_file_id, "completed batch has no output_file_id"
usage: Final = fetched.usage
assert usage is not None and usage.input_tokens and usage.output_tokens, (
f"completed batch has no batch token counts: {usage!r}"
)
assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), (
f"chat token counts disagree with the batch token counts: {usage!r}"
)
downloaded = client.proxy.transport.download(
f"/v1/files/{fetched.output_file_id}/content",

View file

@ -70,10 +70,11 @@ def _batch_cost_result(
"""Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns,
for mocking it in tests that only care about cost/usage/models."""
from litellm.batches.batch_utils import BatchCostUsageResult
from litellm.types.utils import Usage
return BatchCostUsageResult(
cost=cost,
usage=usage,
usage=Usage(**usage),
models=models,
successful_requests=successful_requests,
failed_requests=failed_requests,
@ -3094,7 +3095,7 @@ class TestMultiPodBatchCostClaim:
patch(
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
new_callable=AsyncMock,
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]),
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]),
),
patch(
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
@ -3509,7 +3510,17 @@ class TestMultiPodBatchCostClaim:
assert journal == ["fetch", "claim", "bill"]
assert row.status == "complete"
assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5}
assert json.loads(row.file_object)["usage"] == {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
"prompt_tokens_details": None,
"completion_tokens_details": None,
"input_tokens": 10,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": 0},
}
logging_obj.async_success_handler.assert_awaited_once()
assert row.batch_processed is True

View file

@ -25,11 +25,13 @@ from litellm.types.utils import (
HiddenParams,
ImageObject,
ImageResponse,
LiteLLMBatch,
ModelResponse,
ModelResponseStream,
TextCompletionResponse,
all_litellm_params,
is_decisions_model_mode,
Usage,
text_tokens_without_nested_reasoning,
)
@ -1007,3 +1009,137 @@ def test_text_tokens_without_nested_reasoning_clamps(
)
== expected_text_tokens
)
_BATCH_FIELDS: Final = {
"id": "batch_abc",
"object": "batch",
"endpoint": "/v1/responses",
"input_file_id": "file-abc",
"completion_window": "24h",
"status": "completed",
"created_at": 1,
}
_PROVIDER_BATCH_USAGE: Final = {
"input_tokens": 21,
"input_tokens_details": {"cached_tokens": 5},
"output_tokens": 543,
"output_tokens_details": {"reasoning_tokens": 448},
"total_tokens": 564,
}
def _batch_token_counts(batch: LiteLLMBatch) -> tuple[int, int, int, int | None, int | None]:
assert batch.usage is not None
assert batch.usage.prompt_tokens_details is not None
assert batch.usage.completion_tokens_details is not None
return (
batch.usage.prompt_tokens,
batch.usage.completion_tokens,
batch.usage.total_tokens,
batch.usage.prompt_tokens_details.cached_tokens,
batch.usage.completion_tokens_details.reasoning_tokens,
)
@pytest.mark.parametrize(
"usage",
(
pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"),
pytest.param(
{"prompt_tokens": 0, "completion_tokens": 0, **_PROVIDER_BATCH_USAGE},
id="stored_row_with_zeroed_chat_fields",
),
),
)
def test_batch_usage_reads_token_counts_from_the_provider_batch_usage(usage: dict[str, object]) -> None:
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
assert _batch_token_counts(batch) == (21, 543, 564, 5, 448)
assert batch.usage is not None
assert (batch.usage.input_tokens, batch.usage.output_tokens) == (21, 543)
def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None:
batch: Final = LiteLLMBatch.model_validate(
{**_BATCH_FIELDS, "usage": {"prompt_tokens": 28, "completion_tokens": 8, "total_tokens": 36}}
)
assert batch.usage is not None
assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36)
@pytest.mark.parametrize(
"usage",
(
pytest.param({"input_tokens": 10, "output_tokens": 5}, id="no_details_or_total"),
pytest.param(
{"input_tokens": 10, "input_tokens_details": None, "output_tokens": 5, "output_tokens_details": None},
id="null_details",
),
pytest.param(
{
"input_tokens": 10,
"input_tokens_details": {"cached_tokens": None},
"output_tokens": 5,
"output_tokens_details": {"reasoning_tokens": None},
"total_tokens": 15,
},
id="null_detail_counts",
),
),
)
def test_batch_usage_reads_token_counts_from_partial_provider_batch_usage(usage: dict[str, object]) -> None:
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
dumped_usage: Final = json.loads(batch.model_dump_json())["usage"]
assert batch.usage is not None
assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (10, 5, 15)
assert (dumped_usage["input_tokens"], dumped_usage["output_tokens"], dumped_usage["total_tokens"]) == (10, 5, 15)
@pytest.mark.parametrize(
"usage",
(
pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"),
pytest.param(
Usage(
prompt_tokens=21,
completion_tokens=543,
total_tokens=564,
prompt_tokens_details={"cached_tokens": 5},
completion_tokens_details={"reasoning_tokens": 448},
),
id="billed_chat_usage",
),
),
)
def test_batch_dump_carries_the_chat_and_provider_batch_usage_fields(usage: dict[str, object] | Usage) -> None:
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
dumped: Final = batch.model_dump_json()
dumped_usage: Final = json.loads(dumped)["usage"]
assert {key: dumped_usage[key] for key in _PROVIDER_BATCH_USAGE} == _PROVIDER_BATCH_USAGE
assert (dumped_usage["prompt_tokens"], dumped_usage["completion_tokens"]) == (21, 543)
assert batch.model_dump()["usage"] == dumped_usage
assert _batch_token_counts(LiteLLMBatch.model_validate_json(dumped)) == (21, 543, 564, 5, 448)
def test_batch_dump_keeps_cache_write_tokens_from_billed_usage() -> None:
batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(
update={"usage": Usage(prompt_tokens=30, completion_tokens=4, total_tokens=34, cache_creation_input_tokens=7)}
)
dumped: Final = batch.model_dump_json()
dumped_usage: Final = json.loads(dumped)["usage"]
assert (dumped_usage["prompt_tokens"], dumped_usage["input_tokens"]) == (30, 30)
assert (dumped_usage["completion_tokens"], dumped_usage["output_tokens"]) == (4, 4)
assert dumped_usage["prompt_tokens_details"]["cache_write_tokens"] == 7
assert None not in batch.model_dump(exclude_none=True)["usage"]["prompt_tokens_details"].values()
assert json.loads(LiteLLMBatch.model_validate_json(dumped).model_dump_json())["usage"] == dumped_usage
def test_batch_json_keeps_usage_that_skipped_validation() -> None:
batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(update={"usage": {"prompt_tokens": 3}})
assert json.loads(batch.model_dump_json(warnings=False))["usage"] == {"prompt_tokens": 3}