mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 59608b3785 into cfc651dcd5
This commit is contained in:
commit
9e062832e5
5 changed files with 269 additions and 4 deletions
|
|
@ -20,6 +20,7 @@ from openai._models import BaseModel as OpenAIObject
|
|||
from openai.types.audio.transcription_create_params import (
|
||||
FileTypes as FileTypes,
|
||||
)
|
||||
from openai.types.batch_usage import BatchUsage, InputTokensDetails, OutputTokensDetails
|
||||
from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion
|
||||
from openai.types.completion_usage import (
|
||||
CompletionTokensDetails,
|
||||
|
|
@ -44,6 +45,7 @@ from pydantic import (
|
|||
JsonValue,
|
||||
PrivateAttr,
|
||||
SkipValidation,
|
||||
ValidationError,
|
||||
field_serializer,
|
||||
field_validator,
|
||||
)
|
||||
|
|
@ -4758,6 +4760,87 @@ class LiteLLMFineTuningJob(FineTuningJob):
|
|||
self._hidden_params = kwargs.get("_hidden_params", {})
|
||||
|
||||
|
||||
class _BatchInputTokensDetails(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
cached_tokens: int | None = None
|
||||
|
||||
|
||||
class _BatchOutputTokensDetails(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
reasoning_tokens: int | None = None
|
||||
|
||||
|
||||
class _BatchTokenCounts(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
total_tokens: int | None = None
|
||||
input_tokens_details: _BatchInputTokensDetails | None = None
|
||||
output_tokens_details: _BatchOutputTokensDetails | None = None
|
||||
|
||||
|
||||
def _parse_batch_token_counts(value: object) -> _BatchTokenCounts | None:
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
try:
|
||||
return _BatchTokenCounts.model_validate(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _prompt_details_with_cached_tokens(
|
||||
details: PromptTokensDetailsWrapper | None, cached_tokens: int | None
|
||||
) -> PromptTokensDetailsWrapper | None:
|
||||
if not cached_tokens:
|
||||
return details
|
||||
return (details or PromptTokensDetailsWrapper()).model_copy(update={"cached_tokens": cached_tokens})
|
||||
|
||||
|
||||
def _completion_details_with_reasoning_tokens(
|
||||
details: CompletionTokensDetailsWrapper | None, reasoning_tokens: int | None
|
||||
) -> CompletionTokensDetailsWrapper | None:
|
||||
if not reasoning_tokens:
|
||||
return details
|
||||
return (details or CompletionTokensDetailsWrapper()).model_copy(update={"reasoning_tokens": reasoning_tokens})
|
||||
|
||||
|
||||
def _usage_with_batch_token_counts(usage: Usage, counts: _BatchTokenCounts) -> Usage:
|
||||
return usage.model_copy(
|
||||
update={
|
||||
"prompt_tokens": counts.input_tokens,
|
||||
"completion_tokens": counts.output_tokens,
|
||||
"total_tokens": (
|
||||
counts.input_tokens + counts.output_tokens if counts.total_tokens is None else counts.total_tokens
|
||||
),
|
||||
"prompt_tokens_details": _prompt_details_with_cached_tokens(
|
||||
usage.prompt_tokens_details,
|
||||
counts.input_tokens_details.cached_tokens if counts.input_tokens_details else None,
|
||||
),
|
||||
"completion_tokens_details": _completion_details_with_reasoning_tokens(
|
||||
usage.completion_tokens_details,
|
||||
counts.output_tokens_details.reasoning_tokens if counts.output_tokens_details else None,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
|
||||
cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None
|
||||
reasoning_tokens: Final = (
|
||||
usage.completion_tokens_details.reasoning_tokens if usage.completion_tokens_details else None
|
||||
)
|
||||
return BatchUsage(
|
||||
input_tokens=usage.prompt_tokens,
|
||||
input_tokens_details=InputTokensDetails(cached_tokens=cached_tokens or 0),
|
||||
output_tokens=usage.completion_tokens,
|
||||
output_tokens_details=OutputTokensDetails(reasoning_tokens=reasoning_tokens or 0),
|
||||
total_tokens=usage.total_tokens,
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMBatch(Batch):
|
||||
_hidden_params: dict = {}
|
||||
|
||||
|
|
@ -4771,6 +4854,23 @@ class LiteLLMBatch(Batch):
|
|||
|
||||
usage: Usage | None = None
|
||||
|
||||
@field_validator("usage", mode="before")
|
||||
@classmethod
|
||||
def _usage_from_batch_token_counts(cls, value: object) -> object:
|
||||
counts: Final = _parse_batch_token_counts(value)
|
||||
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
|
||||
|
||||
@field_serializer("usage")
|
||||
def _serialize_usage_with_batch_fields(
|
||||
self, usage: Usage | None, info: FieldSerializationInfo
|
||||
) -> Mapping[str, object] | None:
|
||||
if not isinstance(usage, Usage):
|
||||
return usage
|
||||
return {
|
||||
**usage.model_dump(mode=info.mode, exclude_none=info.exclude_none),
|
||||
**_batch_usage_from_chat_usage(usage).model_dump(mode=info.mode),
|
||||
}
|
||||
|
||||
def __contains__(self, key) -> bool:
|
||||
# Define custom behavior for the 'in' operator
|
||||
return hasattr(self, key)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from e2e_metadata import step
|
||||
from proxy_client import ProxyClient
|
||||
|
|
@ -77,6 +77,16 @@ class BatchErrorList(BaseModel):
|
|||
data: list[BatchErrorItem] = []
|
||||
|
||||
|
||||
class BatchUsage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
completion_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
|
||||
|
||||
class BatchObject(BaseModel):
|
||||
id: str
|
||||
object: str | None = None
|
||||
|
|
@ -90,6 +100,7 @@ class BatchObject(BaseModel):
|
|||
completion_window: str | None = None
|
||||
created_at: int | None = None
|
||||
model: str | None = None
|
||||
usage: BatchUsage | None = None
|
||||
|
||||
|
||||
class BatchList(BaseModel):
|
||||
|
|
|
|||
|
|
@ -1893,6 +1893,13 @@ class TestBatchTerminalState:
|
|||
f"listed-completed marker retrieved as {fetched.status!r}"
|
||||
)
|
||||
assert fetched.output_file_id, "completed batch has no output_file_id"
|
||||
usage: Final = fetched.usage
|
||||
assert usage is not None and usage.input_tokens and usage.output_tokens, (
|
||||
f"completed batch has no batch token counts: {usage!r}"
|
||||
)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), (
|
||||
f"chat token counts disagree with the batch token counts: {usage!r}"
|
||||
)
|
||||
|
||||
downloaded = client.proxy.transport.download(
|
||||
f"/v1/files/{fetched.output_file_id}/content",
|
||||
|
|
|
|||
|
|
@ -70,10 +70,11 @@ def _batch_cost_result(
|
|||
"""Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns,
|
||||
for mocking it in tests that only care about cost/usage/models."""
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
return BatchCostUsageResult(
|
||||
cost=cost,
|
||||
usage=usage,
|
||||
usage=Usage(**usage),
|
||||
models=models,
|
||||
successful_requests=successful_requests,
|
||||
failed_requests=failed_requests,
|
||||
|
|
@ -3094,7 +3095,7 @@ class TestMultiPodBatchCostClaim:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]),
|
||||
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]),
|
||||
),
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
|
||||
|
|
@ -3509,7 +3510,17 @@ class TestMultiPodBatchCostClaim:
|
|||
|
||||
assert journal == ["fetch", "claim", "bill"]
|
||||
assert row.status == "complete"
|
||||
assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5}
|
||||
assert json.loads(row.file_object)["usage"] == {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
"prompt_tokens_details": None,
|
||||
"completion_tokens_details": None,
|
||||
"input_tokens": 10,
|
||||
"input_tokens_details": {"cached_tokens": 0},
|
||||
"output_tokens": 5,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
}
|
||||
logging_obj.async_success_handler.assert_awaited_once()
|
||||
assert row.batch_processed is True
|
||||
|
||||
|
|
|
|||
|
|
@ -25,11 +25,13 @@ from litellm.types.utils import (
|
|||
HiddenParams,
|
||||
ImageObject,
|
||||
ImageResponse,
|
||||
LiteLLMBatch,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
TextCompletionResponse,
|
||||
all_litellm_params,
|
||||
is_decisions_model_mode,
|
||||
Usage,
|
||||
text_tokens_without_nested_reasoning,
|
||||
)
|
||||
|
||||
|
|
@ -1007,3 +1009,137 @@ def test_text_tokens_without_nested_reasoning_clamps(
|
|||
)
|
||||
== expected_text_tokens
|
||||
)
|
||||
|
||||
|
||||
_BATCH_FIELDS: Final = {
|
||||
"id": "batch_abc",
|
||||
"object": "batch",
|
||||
"endpoint": "/v1/responses",
|
||||
"input_file_id": "file-abc",
|
||||
"completion_window": "24h",
|
||||
"status": "completed",
|
||||
"created_at": 1,
|
||||
}
|
||||
_PROVIDER_BATCH_USAGE: Final = {
|
||||
"input_tokens": 21,
|
||||
"input_tokens_details": {"cached_tokens": 5},
|
||||
"output_tokens": 543,
|
||||
"output_tokens_details": {"reasoning_tokens": 448},
|
||||
"total_tokens": 564,
|
||||
}
|
||||
|
||||
|
||||
def _batch_token_counts(batch: LiteLLMBatch) -> tuple[int, int, int, int | None, int | None]:
|
||||
assert batch.usage is not None
|
||||
assert batch.usage.prompt_tokens_details is not None
|
||||
assert batch.usage.completion_tokens_details is not None
|
||||
return (
|
||||
batch.usage.prompt_tokens,
|
||||
batch.usage.completion_tokens,
|
||||
batch.usage.total_tokens,
|
||||
batch.usage.prompt_tokens_details.cached_tokens,
|
||||
batch.usage.completion_tokens_details.reasoning_tokens,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"usage",
|
||||
(
|
||||
pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"),
|
||||
pytest.param(
|
||||
{"prompt_tokens": 0, "completion_tokens": 0, **_PROVIDER_BATCH_USAGE},
|
||||
id="stored_row_with_zeroed_chat_fields",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_batch_usage_reads_token_counts_from_the_provider_batch_usage(usage: dict[str, object]) -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
|
||||
|
||||
assert _batch_token_counts(batch) == (21, 543, 564, 5, 448)
|
||||
assert batch.usage is not None
|
||||
assert (batch.usage.input_tokens, batch.usage.output_tokens) == (21, 543)
|
||||
|
||||
|
||||
def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate(
|
||||
{**_BATCH_FIELDS, "usage": {"prompt_tokens": 28, "completion_tokens": 8, "total_tokens": 36}}
|
||||
)
|
||||
|
||||
assert batch.usage is not None
|
||||
assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"usage",
|
||||
(
|
||||
pytest.param({"input_tokens": 10, "output_tokens": 5}, id="no_details_or_total"),
|
||||
pytest.param(
|
||||
{"input_tokens": 10, "input_tokens_details": None, "output_tokens": 5, "output_tokens_details": None},
|
||||
id="null_details",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"input_tokens": 10,
|
||||
"input_tokens_details": {"cached_tokens": None},
|
||||
"output_tokens": 5,
|
||||
"output_tokens_details": {"reasoning_tokens": None},
|
||||
"total_tokens": 15,
|
||||
},
|
||||
id="null_detail_counts",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_batch_usage_reads_token_counts_from_partial_provider_batch_usage(usage: dict[str, object]) -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
|
||||
dumped_usage: Final = json.loads(batch.model_dump_json())["usage"]
|
||||
|
||||
assert batch.usage is not None
|
||||
assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (10, 5, 15)
|
||||
assert (dumped_usage["input_tokens"], dumped_usage["output_tokens"], dumped_usage["total_tokens"]) == (10, 5, 15)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"usage",
|
||||
(
|
||||
pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"),
|
||||
pytest.param(
|
||||
Usage(
|
||||
prompt_tokens=21,
|
||||
completion_tokens=543,
|
||||
total_tokens=564,
|
||||
prompt_tokens_details={"cached_tokens": 5},
|
||||
completion_tokens_details={"reasoning_tokens": 448},
|
||||
),
|
||||
id="billed_chat_usage",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_batch_dump_carries_the_chat_and_provider_batch_usage_fields(usage: dict[str, object] | Usage) -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage})
|
||||
dumped: Final = batch.model_dump_json()
|
||||
dumped_usage: Final = json.loads(dumped)["usage"]
|
||||
|
||||
assert {key: dumped_usage[key] for key in _PROVIDER_BATCH_USAGE} == _PROVIDER_BATCH_USAGE
|
||||
assert (dumped_usage["prompt_tokens"], dumped_usage["completion_tokens"]) == (21, 543)
|
||||
assert batch.model_dump()["usage"] == dumped_usage
|
||||
assert _batch_token_counts(LiteLLMBatch.model_validate_json(dumped)) == (21, 543, 564, 5, 448)
|
||||
|
||||
|
||||
def test_batch_dump_keeps_cache_write_tokens_from_billed_usage() -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(
|
||||
update={"usage": Usage(prompt_tokens=30, completion_tokens=4, total_tokens=34, cache_creation_input_tokens=7)}
|
||||
)
|
||||
dumped: Final = batch.model_dump_json()
|
||||
dumped_usage: Final = json.loads(dumped)["usage"]
|
||||
|
||||
assert (dumped_usage["prompt_tokens"], dumped_usage["input_tokens"]) == (30, 30)
|
||||
assert (dumped_usage["completion_tokens"], dumped_usage["output_tokens"]) == (4, 4)
|
||||
assert dumped_usage["prompt_tokens_details"]["cache_write_tokens"] == 7
|
||||
assert None not in batch.model_dump(exclude_none=True)["usage"]["prompt_tokens_details"].values()
|
||||
assert json.loads(LiteLLMBatch.model_validate_json(dumped).model_dump_json())["usage"] == dumped_usage
|
||||
|
||||
|
||||
def test_batch_json_keeps_usage_that_skipped_validation() -> None:
|
||||
batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(update={"usage": {"prompt_tokens": 3}})
|
||||
|
||||
assert json.loads(batch.model_dump_json(warnings=False))["usage"] == {"prompt_tokens": 3}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue