mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
Merge pull request #37208 from BerriAI/litellm_managed_batches_observability
fix(batches): aggregate reasoning tokens and per-line pass/fail counts
This commit is contained in:
Mateo Wang
GitHub
commit
20cfccaf5f
16 changed files with 809 additions and 278 deletions
|
|
@ -2,9 +2,10 @@
|
|||
Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
|
||||
"""
|
||||
|
||||
from dataclasses import replace as dataclasses_replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Tuple, cast
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -626,6 +627,7 @@ class CheckBatchCost:
|
|||
later poll.
|
||||
"""
|
||||
from litellm.batches.batch_utils import (
|
||||
count_error_file_failed_requests,
|
||||
_get_file_content_as_dictionary,
|
||||
calculate_batch_cost_and_usage,
|
||||
)
|
||||
|
|
@ -761,16 +763,33 @@ class CheckBatchCost:
|
|||
model_id=model_id,
|
||||
deployment_model=litellm_model_name,
|
||||
)
|
||||
batch_cost, batch_usage, batch_models = (
|
||||
await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=file_content_as_dict,
|
||||
custom_llm_provider=llm_provider, # type: ignore
|
||||
model_name=model_name,
|
||||
model_info=deployment_model_info,
|
||||
batch_file_provider: Final = cast(
|
||||
Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], llm_provider
|
||||
)
|
||||
output_file_result: Final = await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=file_content_as_dict,
|
||||
custom_llm_provider=batch_file_provider,
|
||||
model_name=model_name,
|
||||
model_info=deployment_model_info,
|
||||
)
|
||||
error_file_failed_requests: Final = await count_error_file_failed_requests(
|
||||
response,
|
||||
custom_llm_provider=batch_file_provider,
|
||||
litellm_params={
|
||||
**credentials,
|
||||
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
|
||||
},
|
||||
)
|
||||
batch_result: Final = (
|
||||
output_file_result
|
||||
if not error_file_failed_requests
|
||||
else dataclasses_replace(
|
||||
output_file_result,
|
||||
failed_requests=output_file_result.failed_requests + error_file_failed_requests,
|
||||
)
|
||||
)
|
||||
logging_obj = LiteLLMLogging(
|
||||
model=batch_models[0],
|
||||
model=batch_result.models[0],
|
||||
messages=[{"role": "user", "content": "<retrieve_batch>"}],
|
||||
stream=False,
|
||||
call_type="aretrieve_batch",
|
||||
|
|
@ -802,9 +821,11 @@ class CheckBatchCost:
|
|||
try:
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_cost,
|
||||
batch_usage=batch_usage,
|
||||
batch_models=batch_models,
|
||||
batch_cost=batch_result.cost,
|
||||
batch_usage=batch_result.usage,
|
||||
batch_models=batch_result.models,
|
||||
batch_successful_requests=batch_result.successful_requests,
|
||||
batch_failed_requests=batch_result.failed_requests,
|
||||
)
|
||||
except Exception:
|
||||
await self._release_job_claim(job)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import replace as dataclasses_replace
|
||||
from enum import Enum
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
import litellm
|
||||
|
|
@ -12,12 +14,23 @@ from litellm.types.utils import CallTypes, ModelInfo, Usage
|
|||
from litellm.utils import token_counter
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchCostUsageResult:
|
||||
"""Aggregate cost, usage, and per-line pass/fail counts for a completed batch."""
|
||||
|
||||
cost: float
|
||||
usage: Usage
|
||||
models: list[str]
|
||||
successful_requests: int
|
||||
failed_requests: int
|
||||
|
||||
|
||||
async def calculate_batch_cost_and_usage(
|
||||
file_content_dictionary: list[dict],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
|
||||
model_name: str | None = None,
|
||||
model_info: ModelInfo | None = None,
|
||||
) -> tuple[float, Usage, list[str]]:
|
||||
) -> BatchCostUsageResult:
|
||||
"""
|
||||
Calculate the cost and usage of a batch.
|
||||
|
||||
|
|
@ -32,8 +45,7 @@ async def calculate_batch_cost_and_usage(
|
|||
and model_name
|
||||
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
|
||||
):
|
||||
batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name)
|
||||
return batch_cost, batch_usage, [model_name]
|
||||
return calculate_vertex_ai_batch_cost_and_usage(file_content_dictionary, model_name)
|
||||
|
||||
return _aggregate_batch_cost_usage_models(
|
||||
entries=file_content_dictionary,
|
||||
|
|
@ -49,7 +61,7 @@ async def _handle_completed_batch(
|
|||
model_name: str | None = None,
|
||||
litellm_params: dict | None = None,
|
||||
model_info: ModelInfo | None = None,
|
||||
) -> tuple[float, Usage, list[str]]:
|
||||
) -> BatchCostUsageResult:
|
||||
"""Fetch a completed batch's output file and aggregate its cost, usage, and
|
||||
models in a single pass over the JSONL lines, so the parsed file content is
|
||||
never materialized in memory.
|
||||
|
|
@ -72,27 +84,49 @@ async def _handle_completed_batch(
|
|||
# The generic retrieval helper keeps raising for callers that explicitly ask
|
||||
# for a missing output file.
|
||||
if batch.output_file_id is None:
|
||||
return 0.0, Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), []
|
||||
return BatchCostUsageResult(
|
||||
cost=0.0,
|
||||
usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0),
|
||||
models=[], # mutable-ok: no output file means no model was ever priced; BatchCostUsageResult.models requires list[str]
|
||||
successful_requests=0,
|
||||
failed_requests=await count_error_file_failed_requests(
|
||||
batch, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
|
||||
),
|
||||
)
|
||||
|
||||
file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params)
|
||||
|
||||
if (
|
||||
custom_llm_provider == "vertex_ai"
|
||||
and model_name
|
||||
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
|
||||
):
|
||||
batch_cost, batch_usage = calculate_vertex_ai_batch_cost_and_usage(
|
||||
_get_file_content_as_dictionary(file_content), model_name
|
||||
)
|
||||
return batch_cost, batch_usage, [model_name]
|
||||
|
||||
return _aggregate_batch_cost_usage_models(
|
||||
entries=_iter_batch_output_entries(file_content),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_name=model_name,
|
||||
model_info=model_info,
|
||||
error_file_failed_requests: Final = await count_error_file_failed_requests(
|
||||
batch, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
output_file_result: Final = (
|
||||
calculate_vertex_ai_batch_cost_and_usage(_get_file_content_as_dictionary(file_content), model_name)
|
||||
if (
|
||||
custom_llm_provider == "vertex_ai"
|
||||
and model_name
|
||||
and getattr(litellm, "disable_vertex_batch_output_transformation", False)
|
||||
)
|
||||
else _aggregate_batch_cost_usage_models(
|
||||
entries=_iter_batch_output_entries(file_content),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_name=model_name,
|
||||
model_info=model_info,
|
||||
)
|
||||
)
|
||||
|
||||
if not error_file_failed_requests:
|
||||
return output_file_result
|
||||
return dataclasses_replace(
|
||||
output_file_result, failed_requests=output_file_result.failed_requests + error_file_failed_requests
|
||||
)
|
||||
|
||||
|
||||
class _LineOutcome(Enum):
|
||||
"""A batch output line that yielded no billable stats."""
|
||||
|
||||
PROVIDER_FAILED = "provider_failed"
|
||||
UNCOSTABLE = "uncostable"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BatchOutputLineStats:
|
||||
|
|
@ -102,19 +136,27 @@ class _BatchOutputLineStats:
|
|||
total_tokens: int
|
||||
cache_read_tokens: int
|
||||
cache_creation_tokens: int
|
||||
reasoning_tokens: int
|
||||
model: str | None
|
||||
|
||||
|
||||
def _iter_successful_output_line_stats(
|
||||
def _classify_output_line_stats(
|
||||
entries: Iterable[dict],
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
|
||||
model_name: str | None,
|
||||
model_info: ModelInfo | None,
|
||||
) -> Iterator[_BatchOutputLineStats]:
|
||||
) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
|
||||
"""Classify every output line in a single pass, so counting failures never needs
|
||||
a second read of a potentially huge output file. A line the provider reported as
|
||||
failed yields ``PROVIDER_FAILED``; a successful line litellm could not price
|
||||
yields ``UNCOSTABLE`` and still counts as a successful request billed at $0, so
|
||||
the counts stay reconcilable with the provider's own ``request_counts``."""
|
||||
for entry in entries:
|
||||
if not _batch_response_was_successful(entry, custom_llm_provider):
|
||||
yield _LineOutcome.PROVIDER_FAILED
|
||||
continue
|
||||
stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
|
||||
if stats is not None:
|
||||
yield stats
|
||||
yield stats if stats is not None else _LineOutcome.UNCOSTABLE
|
||||
|
||||
|
||||
def _safe_output_line_stats(
|
||||
|
|
@ -123,13 +165,11 @@ def _safe_output_line_stats(
|
|||
model_name: str | None,
|
||||
model_info: ModelInfo | None,
|
||||
) -> _BatchOutputLineStats | None:
|
||||
"""Return the stats for one batch output line, or None for a line that is
|
||||
unsuccessful or cannot be costed, so a single bad line never aborts the
|
||||
whole batch's cost accounting."""
|
||||
"""Return the stats for one provider-successful batch output line, or None when
|
||||
it cannot be costed, so a single bad line never aborts the whole batch's cost
|
||||
accounting."""
|
||||
custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
|
||||
try:
|
||||
if not _batch_response_was_successful(entry, custom_llm_provider):
|
||||
return None
|
||||
return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
|
||||
except Exception as e: # noqa: BLE001 # any single line's costing failure must not abort the whole batch
|
||||
verbose_logger.warning(
|
||||
|
|
@ -152,6 +192,7 @@ def _compute_output_line_stats(
|
|||
prompt_details: Final = parse_prompt_tokens_details(usage)
|
||||
raw_model: Final = response_body.get("model")
|
||||
response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
|
||||
completion_details: Final = usage.completion_tokens_details
|
||||
return _BatchOutputLineStats(
|
||||
cost=_output_line_cost(
|
||||
response_body=response_body,
|
||||
|
|
@ -166,6 +207,7 @@ def _compute_output_line_stats(
|
|||
total_tokens=usage.total_tokens,
|
||||
cache_read_tokens=prompt_details["cache_hit_tokens"],
|
||||
cache_creation_tokens=prompt_details["cache_creation_tokens"],
|
||||
reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
|
||||
model=response_model,
|
||||
)
|
||||
|
||||
|
|
@ -203,10 +245,14 @@ def _aggregate_batch_cost_usage_models(
|
|||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
|
||||
model_name: str | None = None,
|
||||
model_info: ModelInfo | None = None,
|
||||
) -> tuple[float, Usage, list[str]]:
|
||||
"""Aggregate cost, usage, and models from batch output entries in a single
|
||||
pass, holding one small stats record per line instead of the parsed file."""
|
||||
line_stats: Final = tuple(_iter_successful_output_line_stats(entries, custom_llm_provider, model_name, model_info))
|
||||
) -> BatchCostUsageResult:
|
||||
"""Aggregate cost, usage, models, and pass/fail counts from batch output
|
||||
entries in a single pass, holding one small stats record per line instead
|
||||
of the parsed file."""
|
||||
all_results: Final = tuple(_classify_output_line_stats(entries, custom_llm_provider, model_name, model_info))
|
||||
line_stats: Final = tuple(result for result in all_results if isinstance(result, _BatchOutputLineStats))
|
||||
failed_requests: Final = sum(1 for result in all_results if result is _LineOutcome.PROVIDER_FAILED)
|
||||
successful_requests: Final = len(all_results) - failed_requests
|
||||
|
||||
cache_token_params: Final = {
|
||||
key: tokens
|
||||
|
|
@ -220,18 +266,32 @@ def _aggregate_batch_cost_usage_models(
|
|||
total_tokens=sum(stats.total_tokens for stats in line_stats),
|
||||
prompt_tokens=sum(stats.prompt_tokens for stats in line_stats),
|
||||
completion_tokens=sum(stats.completion_tokens for stats in line_stats),
|
||||
reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
|
||||
**cache_token_params,
|
||||
)
|
||||
batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
|
||||
total_cost: Final = sum((stats.cost for stats in line_stats), 0.0)
|
||||
verbose_logger.debug("batch output aggregate: cost=%s usage=%s models=%s", total_cost, batch_usage, batch_models)
|
||||
return total_cost, batch_usage, batch_models
|
||||
verbose_logger.debug(
|
||||
"batch output aggregate: cost=%s usage=%s models=%s successful=%d failed=%d",
|
||||
total_cost,
|
||||
batch_usage,
|
||||
batch_models,
|
||||
successful_requests,
|
||||
failed_requests,
|
||||
)
|
||||
return BatchCostUsageResult(
|
||||
cost=total_cost,
|
||||
usage=batch_usage,
|
||||
models=batch_models,
|
||||
successful_requests=successful_requests,
|
||||
failed_requests=failed_requests,
|
||||
)
|
||||
|
||||
|
||||
def calculate_vertex_ai_batch_cost_and_usage(
|
||||
vertex_ai_batch_responses: list[dict],
|
||||
model_name: str | None = None,
|
||||
) -> tuple[float, Usage]:
|
||||
) -> BatchCostUsageResult:
|
||||
"""
|
||||
Calculate both cost and usage from raw Vertex AI batch responses.
|
||||
|
||||
|
|
@ -242,6 +302,10 @@ def calculate_vertex_ai_batch_cost_and_usage(
|
|||
{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}}}
|
||||
|
||||
usageMetadata contains promptTokenCount, candidatesTokenCount, totalTokenCount.
|
||||
|
||||
A row with no ``response`` is counted as failed - the same signal already
|
||||
used to skip it from cost/usage aggregation, since Vertex batch prediction
|
||||
output doesn't establish a distinct error shape in this (non-default) path.
|
||||
"""
|
||||
from litellm.cost_calculator import batch_cost_calculator
|
||||
|
||||
|
|
@ -249,12 +313,16 @@ def calculate_vertex_ai_batch_cost_and_usage(
|
|||
total_tokens = 0
|
||||
prompt_tokens = 0
|
||||
completion_tokens = 0
|
||||
successful_requests = 0 # rebind-ok: loop accumulator, matches total_cost/total_tokens above
|
||||
failed_requests = 0 # rebind-ok: loop accumulator, matches total_cost/total_tokens above
|
||||
actual_model_name: Final = model_name or "gemini-2.0-flash-001"
|
||||
|
||||
for response in vertex_ai_batch_responses:
|
||||
response_body = response.get("response")
|
||||
if response_body is None:
|
||||
failed_requests += 1
|
||||
continue
|
||||
successful_requests += 1
|
||||
|
||||
usage_metadata = response_body.get("usageMetadata", {})
|
||||
_prompt = usage_metadata.get("promptTokenCount", 0) or 0
|
||||
|
|
@ -282,17 +350,25 @@ def calculate_vertex_ai_batch_cost_and_usage(
|
|||
total_tokens += _total
|
||||
|
||||
verbose_logger.info(
|
||||
"vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d",
|
||||
"vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d, successful=%d, failed=%d",
|
||||
total_cost,
|
||||
prompt_tokens,
|
||||
completion_tokens,
|
||||
total_tokens,
|
||||
successful_requests,
|
||||
failed_requests,
|
||||
)
|
||||
|
||||
return total_cost, Usage(
|
||||
total_tokens=total_tokens,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
return BatchCostUsageResult(
|
||||
cost=total_cost,
|
||||
usage=Usage(
|
||||
total_tokens=total_tokens,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
),
|
||||
models=[actual_model_name],
|
||||
successful_requests=successful_requests,
|
||||
failed_requests=failed_requests,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -322,6 +398,36 @@ def _provider_output_file_id(output_file_id: str) -> str:
|
|||
return extracted
|
||||
|
||||
|
||||
async def _fetch_batch_managed_file_content(
|
||||
file_id: str,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
|
||||
litellm_params: dict | None = None,
|
||||
) -> bytes:
|
||||
"""
|
||||
Fetch a batch's output or error file and return its raw JSONL bytes.
|
||||
|
||||
Args:
|
||||
file_id: The provider or unified (litellm-managed) file id to fetch
|
||||
custom_llm_provider: The LLM provider
|
||||
litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
|
||||
Required for Azure and other providers that need authentication
|
||||
"""
|
||||
from litellm.files.main import afile_content
|
||||
|
||||
# Build kwargs for afile_content with credentials from litellm_params
|
||||
file_content_kwargs: Final = {
|
||||
"file_id": _provider_output_file_id(file_id),
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
}
|
||||
|
||||
# Extract and add credentials for file access
|
||||
credentials: Final = _extract_file_access_credentials(litellm_params)
|
||||
file_content_kwargs.update(credentials)
|
||||
|
||||
_file_content: Final = await afile_content(**file_content_kwargs)
|
||||
return _file_content.content
|
||||
|
||||
|
||||
async def _fetch_batch_output_file_content(
|
||||
batch: Batch,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
|
||||
|
|
@ -336,25 +442,36 @@ async def _fetch_batch_output_file_content(
|
|||
litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
|
||||
Required for Azure and other providers that need authentication
|
||||
"""
|
||||
from litellm.files.main import afile_content
|
||||
|
||||
if batch.output_file_id is None:
|
||||
raise ValueError("Output file id is None cannot retrieve file content")
|
||||
|
||||
file_id: Final = _provider_output_file_id(batch.output_file_id)
|
||||
return await _fetch_batch_managed_file_content(
|
||||
batch.output_file_id, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
|
||||
)
|
||||
|
||||
# Build kwargs for afile_content with credentials from litellm_params
|
||||
file_content_kwargs: Final = {
|
||||
"file_id": file_id,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
}
|
||||
|
||||
# Extract and add credentials for file access
|
||||
credentials: Final = _extract_file_access_credentials(litellm_params)
|
||||
file_content_kwargs.update(credentials)
|
||||
async def count_error_file_failed_requests(
|
||||
batch: Batch,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
|
||||
litellm_params: dict | None,
|
||||
) -> int:
|
||||
"""Count failed requests reported only in the batch's separate error file.
|
||||
|
||||
_file_content: Final = await afile_content(**file_content_kwargs)
|
||||
return _file_content.content
|
||||
OpenAI-shaped batch providers write successful lines to ``output_file_id``
|
||||
and per-request failures (e.g. a rejected param) to a distinct
|
||||
``error_file_id`` - they never appear in the output file at all, so
|
||||
counting failures from the output file alone silently undercounts them.
|
||||
"""
|
||||
if batch.error_file_id is None:
|
||||
return 0
|
||||
try:
|
||||
error_file_content = await _fetch_batch_managed_file_content(
|
||||
batch.error_file_id, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a failed/missing error file must not abort cost tracking for the batch
|
||||
verbose_logger.debug("Failed to fetch batch error file %s: %s", batch.error_file_id, e)
|
||||
return 0
|
||||
return sum(1 for _ in _iter_batch_input_lines(error_file_content))
|
||||
|
||||
|
||||
def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
|
||||
|
|
|
|||
|
|
@ -2872,6 +2872,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
batch_cost: Final = kwargs.get("batch_cost", None)
|
||||
batch_usage = kwargs.get("batch_usage", None)
|
||||
batch_models = kwargs.get("batch_models", None)
|
||||
batch_successful_requests: Final = kwargs.get("batch_successful_requests", None)
|
||||
batch_failed_requests: Final = kwargs.get("batch_failed_requests", None)
|
||||
has_explicit_batch_data: Final = all(x is not None for x in (batch_cost, batch_usage, batch_models))
|
||||
|
||||
should_compute_batch_data: Final = (
|
||||
|
|
@ -2880,14 +2882,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if has_explicit_batch_data:
|
||||
result._hidden_params["response_cost"] = batch_cost
|
||||
result._hidden_params["batch_models"] = batch_models
|
||||
result._hidden_params["batch_successful_requests"] = batch_successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above
|
||||
result._hidden_params["batch_failed_requests"] = batch_failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
|
||||
result.usage = batch_usage
|
||||
|
||||
elif should_compute_batch_data:
|
||||
(
|
||||
response_cost,
|
||||
batch_usage,
|
||||
batch_models,
|
||||
) = await _handle_completed_batch(
|
||||
batch_result: Final = await _handle_completed_batch(
|
||||
batch=result,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
model_name=self.get_deployment_model_for_cost(),
|
||||
|
|
@ -2895,9 +2895,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model_info=self.get_router_deployment_model_info(),
|
||||
)
|
||||
|
||||
result._hidden_params["response_cost"] = response_cost
|
||||
result._hidden_params["batch_models"] = batch_models
|
||||
result.usage = batch_usage
|
||||
result._hidden_params["response_cost"] = batch_result.cost
|
||||
result._hidden_params["batch_models"] = batch_result.models
|
||||
result._hidden_params["batch_successful_requests"] = batch_result.successful_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
|
||||
result._hidden_params["batch_failed_requests"] = batch_result.failed_requests # pyright: ignore[reportPrivateUsage] # rebind-ok: same pattern as above
|
||||
result.usage = batch_result.usage
|
||||
|
||||
start_time, end_time, result = self._success_handler_helper_fn(
|
||||
start_time=start_time,
|
||||
|
|
@ -5422,6 +5424,8 @@ class StandardLoggingPayloadSetup:
|
|||
additional_headers=None,
|
||||
litellm_overhead_time_ms=None,
|
||||
batch_models=None,
|
||||
batch_successful_requests=None,
|
||||
batch_failed_requests=None,
|
||||
litellm_model_name=None,
|
||||
usage_object=None,
|
||||
)
|
||||
|
|
@ -5812,6 +5816,8 @@ def _extract_response_obj_and_hidden_params(
|
|||
response_cost=None,
|
||||
litellm_overhead_time_ms=None,
|
||||
batch_models=None,
|
||||
batch_successful_requests=None,
|
||||
batch_failed_requests=None,
|
||||
litellm_model_name=None,
|
||||
usage_object=None,
|
||||
)
|
||||
|
|
@ -6228,6 +6234,8 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
|
|||
additional_headers=None,
|
||||
litellm_overhead_time_ms=None,
|
||||
batch_models=None,
|
||||
batch_successful_requests=None,
|
||||
batch_failed_requests=None,
|
||||
litellm_model_name=None,
|
||||
usage_object=None,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3574,6 +3574,8 @@ class SpendLogsMetadata(TypedDict):
|
|||
status: StandardLoggingPayloadStatus
|
||||
proxy_server_request: str | None
|
||||
batch_models: list[str] | None
|
||||
batch_successful_requests: int | None # writable-ok: built by assignment like every sibling key in this TypedDict
|
||||
batch_failed_requests: int | None # writable-ok: built by assignment like every sibling key in this TypedDict
|
||||
error_information: StandardLoggingPayloadErrorInformation | None
|
||||
usage_object: dict | None
|
||||
model_map_information: StandardLoggingModelInformation | None
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ def _get_spend_logs_metadata(
|
|||
metadata: dict | None,
|
||||
applied_guardrails: list[str] | None = None,
|
||||
batch_models: list[str] | None = None,
|
||||
batch_successful_requests: int | None = None,
|
||||
batch_failed_requests: int | None = None,
|
||||
mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None,
|
||||
vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None = None,
|
||||
guardrail_information: list[StandardLoggingGuardrailInformation] | None = None,
|
||||
|
|
@ -121,6 +123,8 @@ def _get_spend_logs_metadata(
|
|||
error_information=None,
|
||||
proxy_server_request=None,
|
||||
batch_models=None,
|
||||
batch_successful_requests=None,
|
||||
batch_failed_requests=None,
|
||||
mcp_tool_call_metadata=None,
|
||||
vector_store_request_metadata=None,
|
||||
model_map_information=None,
|
||||
|
|
@ -154,6 +158,8 @@ def _get_spend_logs_metadata(
|
|||
clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted)
|
||||
clean_metadata["applied_guardrails"] = applied_guardrails
|
||||
clean_metadata["batch_models"] = batch_models
|
||||
clean_metadata["batch_successful_requests"] = batch_successful_requests
|
||||
clean_metadata["batch_failed_requests"] = batch_failed_requests
|
||||
clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
|
||||
clean_metadata["vector_store_request_metadata"] = _get_vector_store_request_for_spend_logs_payload(
|
||||
vector_store_request_metadata
|
||||
|
|
@ -360,6 +366,16 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
|
|||
if standard_logging_payload is not None
|
||||
else None
|
||||
),
|
||||
batch_successful_requests=(
|
||||
standard_logging_payload.get("hidden_params", {}).get("batch_successful_requests", None)
|
||||
if standard_logging_payload is not None
|
||||
else None
|
||||
),
|
||||
batch_failed_requests=(
|
||||
standard_logging_payload.get("hidden_params", {}).get("batch_failed_requests", None)
|
||||
if standard_logging_payload is not None
|
||||
else None
|
||||
),
|
||||
mcp_tool_call_metadata=(
|
||||
standard_logging_payload["metadata"].get("mcp_tool_call_metadata", None)
|
||||
if standard_logging_payload is not None
|
||||
|
|
|
|||
|
|
@ -2957,6 +2957,8 @@ class StandardLoggingHiddenParams(TypedDict):
|
|||
litellm_overhead_time_ms: float | None
|
||||
additional_headers: StandardLoggingAdditionalHeaders | None
|
||||
batch_models: list[str] | None
|
||||
batch_successful_requests: ReadOnly[int | None]
|
||||
batch_failed_requests: ReadOnly[int | None]
|
||||
litellm_model_name: str | None # the model name sent to the provider by litellm
|
||||
usage_object: dict | None
|
||||
|
||||
|
|
|
|||
|
|
@ -116,16 +116,16 @@ def test_aggregate_batch_cost_uses_custom_model_info():
|
|||
"""_aggregate_batch_cost_usage_models should thread model_info to batch_cost_calculator."""
|
||||
file_content = [_make_batch_output_line(prompt_tokens=10, completion_tokens=5)]
|
||||
|
||||
cost, _, _ = _aggregate_batch_cost_usage_models(
|
||||
result = _aggregate_batch_cost_usage_models(
|
||||
entries=file_content,
|
||||
custom_llm_provider="openai",
|
||||
model_info=CUSTOM_MODEL_INFO,
|
||||
)
|
||||
|
||||
expected = (10 * 0.00125) + (5 * 0.005)
|
||||
assert cost == pytest.approx(
|
||||
assert result.cost == pytest.approx(
|
||||
expected
|
||||
), f"Expected total cost {expected}, got {cost}"
|
||||
), f"Expected total cost {expected}, got {result.cost}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data_residency", ["eu", "us"])
|
||||
|
|
@ -164,15 +164,15 @@ async def test_calculate_batch_cost_and_usage_uses_custom_model_info():
|
|||
"""calculate_batch_cost_and_usage should thread model_info."""
|
||||
file_content = [_make_batch_output_line(prompt_tokens=10, completion_tokens=5)]
|
||||
|
||||
batch_cost, batch_usage, batch_models = await calculate_batch_cost_and_usage(
|
||||
result = await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=file_content,
|
||||
custom_llm_provider="openai",
|
||||
model_info=CUSTOM_MODEL_INFO,
|
||||
)
|
||||
|
||||
expected = (10 * 0.00125) + (5 * 0.005)
|
||||
assert batch_cost == pytest.approx(
|
||||
assert result.cost == pytest.approx(
|
||||
expected
|
||||
), f"Expected total cost {expected}, got {batch_cost}"
|
||||
assert batch_usage.prompt_tokens == 10
|
||||
assert batch_usage.completion_tokens == 5
|
||||
), f"Expected total cost {expected}, got {result.cost}"
|
||||
assert result.usage.prompt_tokens == 10
|
||||
assert result.usage.completion_tokens == 5
|
||||
|
|
|
|||
|
|
@ -1027,7 +1027,7 @@ async def test_batch_logging_azure_credentials_regression():
|
|||
with patch(
|
||||
"litellm.files.main.afile_content", side_effect=mock_afile_content_tracker
|
||||
):
|
||||
cost, usage, models = await _handle_completed_batch(
|
||||
result = await _handle_completed_batch(
|
||||
batch=mock_batch,
|
||||
custom_llm_provider="azure",
|
||||
litellm_params=azure_credentials,
|
||||
|
|
@ -1039,13 +1039,13 @@ async def test_batch_logging_azure_credentials_regression():
|
|||
], "REGRESSION: Credentials not passed through _handle_completed_batch"
|
||||
|
||||
# Verify cost and usage were calculated
|
||||
assert cost > 0, "Cost should be calculated"
|
||||
assert usage.total_tokens == 40, "Usage should be calculated correctly"
|
||||
assert result.cost > 0, "Cost should be calculated"
|
||||
assert result.usage.total_tokens == 40, "Usage should be calculated correctly"
|
||||
|
||||
print(" ✓ Credentials passed through full flow")
|
||||
print(f" ✓ Cost: {cost}")
|
||||
print(f" ✓ Usage: {usage.total_tokens} tokens")
|
||||
print(f" ✓ Models: {models}")
|
||||
print(f" ✓ Cost: {result.cost}")
|
||||
print(f" ✓ Usage: {result.usage.total_tokens} tokens")
|
||||
print(f" ✓ Models: {result.models}")
|
||||
|
||||
# Test 4: Verify error prevention
|
||||
print("\n4. Testing 'Missing credentials' error prevention...")
|
||||
|
|
@ -1064,7 +1064,7 @@ async def test_batch_logging_azure_credentials_regression():
|
|||
"litellm.files.main.afile_content", side_effect=mock_afile_content_tracker
|
||||
):
|
||||
try:
|
||||
cost, usage, models = await _handle_completed_batch(
|
||||
result = await _handle_completed_batch(
|
||||
batch=mock_batch,
|
||||
custom_llm_provider="azure",
|
||||
litellm_params=azure_credentials,
|
||||
|
|
|
|||
|
|
@ -133,12 +133,12 @@ def test_get_file_content_as_dictionary(sample_file_content):
|
|||
|
||||
def test_get_batch_job_total_usage_from_file_content(sample_file_content_dict):
|
||||
with patch("litellm.completion_cost", return_value=0.0):
|
||||
_, usage, _ = _aggregate_batch_cost_usage_models(
|
||||
result = _aggregate_batch_cost_usage_models(
|
||||
entries=sample_file_content_dict, custom_llm_provider="openai"
|
||||
)
|
||||
assert usage.total_tokens == 62 # 30 + 32
|
||||
assert usage.prompt_tokens == 42 # 20 + 22
|
||||
assert usage.completion_tokens == 20 # 10 + 10
|
||||
assert result.usage.total_tokens == 62 # 30 + 32
|
||||
assert result.usage.prompt_tokens == 42 # 20 + 22
|
||||
assert result.usage.completion_tokens == 20 # 10 + 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -151,11 +151,11 @@ async def test_batch_cost_calculator(sample_file_content_dict):
|
|||
so we expect the cost to be 0.5 * 2 = 1.0
|
||||
"""
|
||||
with patch("litellm.completion_cost", return_value=0.5):
|
||||
cost, _, _ = _aggregate_batch_cost_usage_models(
|
||||
result = _aggregate_batch_cost_usage_models(
|
||||
entries=sample_file_content_dict,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
assert cost == 1.0 # 0.5 * 2 successful responses
|
||||
assert result.cost == 1.0 # 0.5 * 2 successful responses
|
||||
|
||||
|
||||
def test_get_response_from_batch_job_output_file(sample_file_content_dict):
|
||||
|
|
@ -221,6 +221,8 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos
|
|||
logging_obj.custom_llm_provider = "openai"
|
||||
|
||||
# Mock _handle_completed_batch to return cost data
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
|
||||
expected_cost = 0.05
|
||||
expected_usage = litellm.Usage(
|
||||
prompt_tokens=100,
|
||||
|
|
@ -231,7 +233,15 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos
|
|||
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.litellm_logging._handle_completed_batch",
|
||||
new=AsyncMock(return_value=(expected_cost, expected_usage, expected_models)),
|
||||
new=AsyncMock(
|
||||
return_value=BatchCostUsageResult(
|
||||
cost=expected_cost,
|
||||
usage=expected_usage,
|
||||
models=expected_models,
|
||||
successful_requests=10,
|
||||
failed_requests=0,
|
||||
)
|
||||
),
|
||||
) as mock_handle_batch:
|
||||
# Call async_success_handler
|
||||
await logging_obj.async_success_handler(
|
||||
|
|
@ -246,6 +256,8 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos
|
|||
# Verify cost and usage were set on the batch result
|
||||
assert mock_batch._hidden_params["response_cost"] == expected_cost
|
||||
assert mock_batch._hidden_params["batch_models"] == expected_models
|
||||
assert mock_batch._hidden_params["batch_successful_requests"] == 10
|
||||
assert mock_batch._hidden_params["batch_failed_requests"] == 0
|
||||
assert mock_batch.usage == expected_usage
|
||||
|
||||
|
||||
|
|
@ -279,7 +291,7 @@ async def test_handle_completed_batch_computes_real_cost_from_output_file(
|
|||
"litellm.batches.batch_utils._fetch_batch_output_file_content",
|
||||
new=AsyncMock(return_value=sample_file_content_bytes),
|
||||
):
|
||||
cost, usage, models = await _handle_completed_batch(
|
||||
result = await _handle_completed_batch(
|
||||
batch=batch, custom_llm_provider="openai"
|
||||
)
|
||||
|
||||
|
|
@ -289,16 +301,18 @@ async def test_handle_completed_batch_computes_real_cost_from_output_file(
|
|||
+ 20 * pricing["output_cost_per_token_batches"]
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(expected_cost)
|
||||
assert cost > 0
|
||||
assert result.cost == pytest.approx(expected_cost)
|
||||
assert result.cost > 0
|
||||
assert (
|
||||
cost
|
||||
result.cost
|
||||
< 42 * pricing["input_cost_per_token"] + 20 * pricing["output_cost_per_token"]
|
||||
)
|
||||
assert usage.prompt_tokens == 42
|
||||
assert usage.completion_tokens == 20
|
||||
assert usage.total_tokens == 62
|
||||
assert models == ["gpt-4o-mini-2024-07-18", "gpt-4o-mini-2024-07-18"]
|
||||
assert result.usage.prompt_tokens == 42
|
||||
assert result.usage.completion_tokens == 20
|
||||
assert result.usage.total_tokens == 62
|
||||
assert result.models == ["gpt-4o-mini-2024-07-18", "gpt-4o-mini-2024-07-18"]
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -537,9 +551,19 @@ async def test_batch_retrieve_cost_tracking_with_partial_explicit_data():
|
|||
)
|
||||
expected_models = ["gpt-5-mini"]
|
||||
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.litellm_logging._handle_completed_batch",
|
||||
new=AsyncMock(return_value=(expected_cost, expected_usage, expected_models)),
|
||||
new=AsyncMock(
|
||||
return_value=BatchCostUsageResult(
|
||||
cost=expected_cost,
|
||||
usage=expected_usage,
|
||||
models=expected_models,
|
||||
successful_requests=8,
|
||||
failed_requests=0,
|
||||
)
|
||||
),
|
||||
) as mock_handle_batch:
|
||||
# Call async_success_handler with partial explicit data
|
||||
await logging_obj.async_success_handler(
|
||||
|
|
@ -555,4 +579,6 @@ async def test_batch_retrieve_cost_tracking_with_partial_explicit_data():
|
|||
# Verify computed cost data was used (not partial explicit data)
|
||||
assert mock_batch._hidden_params["response_cost"] == expected_cost
|
||||
assert mock_batch._hidden_params["batch_models"] == expected_models
|
||||
assert mock_batch._hidden_params["batch_successful_requests"] == 8
|
||||
assert mock_batch._hidden_params["batch_failed_requests"] == 0
|
||||
assert mock_batch.usage == expected_usage
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
"user": "",
|
||||
"team_id": "",
|
||||
"organization_id": "",
|
||||
"metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
|
||||
"metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
|
|||
|
|
@ -9,16 +9,40 @@ ARN unified_object_id) batches with no managed unified id.
|
|||
import asyncio
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
|
||||
_IS_B64 = "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id"
|
||||
_CLAIM_UNIFIED_BATCH_ID = "dW5pZmllZF9iYXRjaF9pZA=="
|
||||
_CLAIM_OUTPUT_FILE_ID = "file-output-123"
|
||||
|
||||
|
||||
def _batch_cost_result(
|
||||
cost: float,
|
||||
usage: dict,
|
||||
models: list[str],
|
||||
successful_requests: int = 1,
|
||||
failed_requests: int = 0,
|
||||
) -> "BatchCostUsageResult":
|
||||
"""Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns,
|
||||
for mocking it in tests that only care about cost/usage/models."""
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
|
||||
return BatchCostUsageResult(
|
||||
cost=cost,
|
||||
usage=usage,
|
||||
models=models,
|
||||
successful_requests=successful_requests,
|
||||
failed_requests=failed_requests,
|
||||
)
|
||||
|
||||
|
||||
def _unmanaged_vertex_file_object(
|
||||
input_file_id="gs://bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash/abc.jsonl",
|
||||
status="validating",
|
||||
|
|
@ -327,7 +351,7 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.01,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["gpt-4"],
|
||||
|
|
@ -432,7 +456,7 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["claude-haiku-4-5"]),
|
||||
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["claude-haiku-4-5"]),
|
||||
),
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
|
||||
|
|
@ -535,7 +559,9 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(0.0052, {"prompt_tokens": 1400, "completion_tokens": 600}, ["claude-haiku-4-5"]),
|
||||
return_value=_batch_cost_result(
|
||||
0.0052, {"prompt_tokens": 1400, "completion_tokens": 600}, ["claude-haiku-4-5"]
|
||||
),
|
||||
) as mock_calculate,
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
|
||||
|
|
@ -634,7 +660,7 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.01,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["gpt-4"],
|
||||
|
|
@ -764,7 +790,7 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.01,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["gpt-4"],
|
||||
|
|
@ -1312,7 +1338,7 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.01,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["gpt-4"],
|
||||
|
|
@ -1347,6 +1373,114 @@ class TestCheckBatchCost:
|
|||
update_data["status"] == terminal_status
|
||||
), f"billed {terminal_status} batch must keep its real terminal status in the DB"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_file_failures_add_to_failed_request_count(
|
||||
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
|
||||
):
|
||||
"""OpenAI-shaped providers report per-request failures only in a separate
|
||||
error file. The poller prices from the output file, so without also counting
|
||||
the error file's lines, batch_failed_requests on the spend log undercounts:
|
||||
regression test for the poller path merging error-file failures.
|
||||
"""
|
||||
import base64
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import respx
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
|
||||
mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
|
||||
mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
|
||||
|
||||
mock_job = MagicMock()
|
||||
mock_job.id = "job-error-file-1"
|
||||
mock_job.unified_object_id = base64.urlsafe_b64encode(
|
||||
b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
|
||||
).decode()
|
||||
mock_job.created_by = "user-1"
|
||||
mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status = "completed"
|
||||
mock_response.output_file_id = "file-output-123"
|
||||
mock_response.error_file_id = "file-error-456"
|
||||
mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
|
||||
mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
|
||||
mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
|
||||
|
||||
mock_deployment = MagicMock()
|
||||
mock_deployment.litellm_params.custom_llm_provider = "openai"
|
||||
mock_deployment.litellm_params.model = "gpt-4"
|
||||
mock_deployment.model_info.model_dump.return_value = {}
|
||||
mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
|
||||
|
||||
succeeded_line = json.dumps(
|
||||
{
|
||||
"custom_id": "req-1",
|
||||
"response": {
|
||||
"status_code": 200,
|
||||
"body": {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"model": "gpt-4",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hi"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
},
|
||||
},
|
||||
},
|
||||
"error": None,
|
||||
}
|
||||
)
|
||||
rejected_line = json.dumps(
|
||||
{
|
||||
"custom_id": "req-2",
|
||||
"response": {
|
||||
"status_code": 400,
|
||||
"body": {"error": {"message": "bad request"}},
|
||||
},
|
||||
"error": None,
|
||||
}
|
||||
)
|
||||
error_file_lines = "\n".join(
|
||||
json.dumps({"custom_id": custom_id, "error": {"message": "rejected"}}) for custom_id in ("req-3", "req-4")
|
||||
)
|
||||
|
||||
with (
|
||||
respx.mock(assert_all_called=True) as provider,
|
||||
patch.object( # test-quality-ok: the poller builds Logging inline, the only seam to its handler kwargs
|
||||
Logging, "async_success_handler", new_callable=AsyncMock
|
||||
) as success_handler,
|
||||
):
|
||||
provider.get("https://api.openai.com/v1/files/file-output-123/content").mock(
|
||||
return_value=httpx.Response(200, content=f"{succeeded_line}\n{rejected_line}\n".encode())
|
||||
)
|
||||
provider.get("https://api.openai.com/v1/files/file-error-456/content").mock(
|
||||
return_value=httpx.Response(200, content=f"{error_file_lines}\n\n".encode())
|
||||
)
|
||||
await check_batch_cost_instance.check_batch_cost()
|
||||
|
||||
spend_log_calls = [call.kwargs for call in success_handler.await_args_list if "batch_cost" in call.kwargs]
|
||||
assert len(spend_log_calls) == 1
|
||||
handler_kwargs = spend_log_calls[0]
|
||||
assert handler_kwargs["batch_successful_requests"] == 1
|
||||
assert handler_kwargs["batch_failed_requests"] == 3, (
|
||||
"2 error-file lines must add to the output file's 1 rejected request"
|
||||
)
|
||||
assert handler_kwargs["batch_models"] == ["gpt-4"]
|
||||
assert handler_kwargs["batch_usage"].total_tokens == 15
|
||||
assert handler_kwargs["batch_cost"] > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_batch_with_missing_output_file_is_retired_unbilled(
|
||||
self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
|
||||
|
|
@ -1518,7 +1652,7 @@ class TestCheckBatchCost:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.01,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["gpt-4"],
|
||||
|
|
@ -1776,7 +1910,7 @@ class TestUnmanagedVertexRouting:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.01,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["gemini-2.5-flash"],
|
||||
|
|
@ -2006,7 +2140,7 @@ class TestUnmanagedBedrockRouting:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(
|
||||
return_value=_batch_cost_result(
|
||||
0.02,
|
||||
{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
["claude-sonnet-4"],
|
||||
|
|
@ -2198,7 +2332,7 @@ class TestManagedOutputFileIdEncodesPublicModelGroup:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(0.01, {"prompt_tokens": 10}, ["gpt-5.5"]),
|
||||
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10}, ["gpt-5.5"]),
|
||||
),
|
||||
patch("litellm.litellm_core_utils.litellm_logging.Logging") as logging_cls,
|
||||
):
|
||||
|
|
@ -2826,7 +2960,7 @@ class TestMultiPodBatchCostClaim:
|
|||
patch(
|
||||
"litellm.batches.batch_utils.calculate_batch_cost_and_usage",
|
||||
new_callable=AsyncMock,
|
||||
return_value=(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]),
|
||||
return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]),
|
||||
),
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider",
|
||||
|
|
|
|||
|
|
@ -211,10 +211,10 @@ def test_estimate_tokens_never_zero_for_short_rows():
|
|||
|
||||
def test_output_models_uses_model_name_override(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
_, _, models = bu._aggregate_batch_cost_usage_models(
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[_success_row(model="ignored")], custom_llm_provider="openai", model_name="forced-model"
|
||||
)
|
||||
assert models == ["forced-model"]
|
||||
assert result.models == ["forced-model"]
|
||||
|
||||
|
||||
def test_output_models_collects_from_successful_only(monkeypatch):
|
||||
|
|
@ -224,15 +224,15 @@ def test_output_models_collects_from_successful_only(monkeypatch):
|
|||
_failed_row(model="should-be-skipped"),
|
||||
_success_row(model="claude-3"),
|
||||
]
|
||||
_, _, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert models == ["gpt-4o", "claude-3"]
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert result.models == ["gpt-4o", "claude-3"]
|
||||
|
||||
|
||||
def test_output_models_skips_successful_without_model(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
rows = [{"response": {"status_code": 200, "body": {}}}]
|
||||
_, _, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert models == []
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert result.models == []
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
@ -399,8 +399,8 @@ def test_total_usage_sums_successful_only(monkeypatch):
|
|||
_failed_row(), # excluded
|
||||
_success_row(usage=_usage(20, 10)), # 30
|
||||
]
|
||||
_, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
|
||||
30,
|
||||
15,
|
||||
45,
|
||||
|
|
@ -418,7 +418,7 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat():
|
|||
)
|
||||
chat_row = _success_row(usage=_usage(10, 5))
|
||||
|
||||
cost, usage, _ = bu._aggregate_batch_cost_usage_models(
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[responses_row, chat_row],
|
||||
custom_llm_provider="openai",
|
||||
model_info={
|
||||
|
|
@ -427,22 +427,79 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat():
|
|||
},
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 30
|
||||
assert usage.completion_tokens == 12
|
||||
assert usage.total_tokens == 42
|
||||
assert usage.cache_read_input_tokens == 3
|
||||
assert cost == pytest.approx((30 * 0.00125) + (12 * 0.005))
|
||||
assert result.usage.prompt_tokens == 30
|
||||
assert result.usage.completion_tokens == 12
|
||||
assert result.usage.total_tokens == 42
|
||||
assert result.usage.cache_read_input_tokens == 3
|
||||
assert result.cost == pytest.approx((30 * 0.00125) + (12 * 0.005))
|
||||
|
||||
|
||||
def test_total_usage_empty_is_zero():
|
||||
cost, usage, models = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai")
|
||||
assert cost == 0.0
|
||||
assert models == []
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai")
|
||||
assert result.cost == 0.0
|
||||
assert result.models == []
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
assert result.successful_requests == 0
|
||||
assert result.failed_requests == 0
|
||||
|
||||
|
||||
def test_total_usage_includes_reasoning_tokens(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
rows = [
|
||||
_success_row(
|
||||
usage={
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 60,
|
||||
"completion_tokens_details": {"reasoning_tokens": 30},
|
||||
}
|
||||
),
|
||||
_success_row(
|
||||
usage={
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 25,
|
||||
"completion_tokens_details": {"reasoning_tokens": 8},
|
||||
}
|
||||
),
|
||||
_failed_row(), # excluded, must not contribute reasoning tokens either
|
||||
]
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert result.usage.completion_tokens_details is not None
|
||||
assert result.usage.completion_tokens_details.reasoning_tokens == 38
|
||||
|
||||
|
||||
def test_aggregate_counts_successful_and_failed_requests(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
rows = [
|
||||
_success_row(usage=_usage(10, 5)),
|
||||
_failed_row(),
|
||||
_success_row(usage=_usage(20, 10)),
|
||||
_failed_row(),
|
||||
_failed_row(),
|
||||
]
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 3
|
||||
assert result.successful_requests + result.failed_requests == len(rows)
|
||||
|
||||
|
||||
def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 1.0)
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[_success_row(usage=_usage(10, 5))], custom_llm_provider="openai"
|
||||
)
|
||||
assert isinstance(result, bu.BatchCostUsageResult)
|
||||
assert (result.cost, result.models, result.successful_requests, result.failed_requests) == (
|
||||
1.0,
|
||||
["gpt-4o"],
|
||||
1,
|
||||
0,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
@ -465,15 +522,22 @@ def test_cost_from_content_completion_cost_path(monkeypatch):
|
|||
_success_row(usage=_usage(20, 10)),
|
||||
]
|
||||
|
||||
total, _, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
|
||||
assert total == 1.0 # 2 successful * 0.5
|
||||
assert result.cost == 1.0 # 2 successful * 0.5
|
||||
assert len(calls) == 2 # failed row not costed
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 1
|
||||
|
||||
|
||||
def test_empty_body_line_does_not_zero_whole_batch():
|
||||
"""A status-200 row with an empty body makes litellm.completion_cost raise;
|
||||
that line must be skipped instead of zeroing the whole batch."""
|
||||
that line must be skipped from pricing instead of zeroing the whole batch.
|
||||
|
||||
The provider still reported it as a success, so it stays in
|
||||
successful_requests and out of failed_requests - otherwise the counts stop
|
||||
reconciling with the provider's own request_counts over a litellm-side
|
||||
pricing gap the customer never caused."""
|
||||
rows = [
|
||||
_success_row(usage=_usage(10, 5)),
|
||||
{
|
||||
|
|
@ -483,11 +547,12 @@ def test_empty_body_line_does_not_zero_whole_batch():
|
|||
_success_row(usage=_usage(20, 10)),
|
||||
]
|
||||
|
||||
cost, usage, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
|
||||
assert cost > 0.0
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (30, 15, 45)
|
||||
assert models == ["gpt-4o", "gpt-4o"]
|
||||
assert result.cost > 0.0
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (30, 15, 45)
|
||||
assert result.models == ["gpt-4o", "gpt-4o"]
|
||||
assert (result.successful_requests, result.failed_requests) == (3, 0)
|
||||
|
||||
|
||||
def test_cost_from_content_model_info_path(monkeypatch):
|
||||
|
|
@ -500,13 +565,13 @@ def test_cost_from_content_model_info_path(monkeypatch):
|
|||
_success_row(usage=_usage(20, 10)),
|
||||
]
|
||||
|
||||
total, _, _ = bu._aggregate_batch_cost_usage_models(
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=rows,
|
||||
custom_llm_provider="openai",
|
||||
model_info={"input_cost_per_token": 0.0}, # type: ignore[arg-type] # truthy -> model_info path
|
||||
)
|
||||
|
||||
assert total == pytest.approx(0.6) # 2 * (0.1 + 0.2)
|
||||
assert result.cost == pytest.approx(0.6) # 2 * (0.1 + 0.2)
|
||||
|
||||
|
||||
def test_aggregate_consumes_entries_in_a_single_pass(monkeypatch):
|
||||
|
|
@ -516,11 +581,13 @@ def test_aggregate_consumes_entries_in_a_single_pass(monkeypatch):
|
|||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.5)
|
||||
one_shot = (row for row in [_success_row(usage=_usage(10, 5)), _failed_row(), _success_row(usage=_usage(20, 10))])
|
||||
|
||||
cost, usage, models = bu._aggregate_batch_cost_usage_models(entries=one_shot, custom_llm_provider="openai")
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=one_shot, custom_llm_provider="openai")
|
||||
|
||||
assert cost == 1.0
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (30, 15, 45)
|
||||
assert models == ["gpt-4o", "gpt-4o"]
|
||||
assert result.cost == 1.0
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (30, 15, 45)
|
||||
assert result.models == ["gpt-4o", "gpt-4o"]
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 1
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
@ -534,7 +601,13 @@ async def test_calculate_vertex_disable_transform_path(monkeypatch):
|
|||
monkeypatch.setattr(
|
||||
bu,
|
||||
"calculate_vertex_ai_batch_cost_and_usage",
|
||||
lambda content, model: (9.9, Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3)),
|
||||
lambda content, model: bu.BatchCostUsageResult(
|
||||
cost=9.9,
|
||||
usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3),
|
||||
models=["gemini-2.0-flash-001"],
|
||||
successful_requests=1,
|
||||
failed_requests=0,
|
||||
),
|
||||
)
|
||||
# generic path must NOT be taken
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -543,12 +616,12 @@ async def test_calculate_vertex_disable_transform_path(monkeypatch):
|
|||
lambda **kw: pytest.fail("generic path should not run"),
|
||||
)
|
||||
|
||||
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
||||
result = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[], custom_llm_provider="vertex_ai", model_name="gemini-2.0-flash-001"
|
||||
)
|
||||
assert cost == 9.9
|
||||
assert usage.total_tokens == 3
|
||||
assert models == ["gemini-2.0-flash-001"]
|
||||
assert result.cost == 9.9
|
||||
assert result.usage.total_tokens == 3
|
||||
assert result.models == ["gemini-2.0-flash-001"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -562,12 +635,12 @@ async def test_calculate_vertex_disable_transform_needs_model_name(monkeypatch):
|
|||
lambda content, model: pytest.fail("raw vertex path should not run"),
|
||||
)
|
||||
|
||||
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
||||
result = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[], custom_llm_provider="vertex_ai"
|
||||
)
|
||||
assert cost == 0.0
|
||||
assert usage.total_tokens == 0
|
||||
assert models == []
|
||||
assert result.cost == 0.0
|
||||
assert result.usage.total_tokens == 0
|
||||
assert result.models == []
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
@ -600,14 +673,16 @@ def test_vertex_cost_and_usage_aggregation(monkeypatch):
|
|||
},
|
||||
]
|
||||
|
||||
cost, usage = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
result = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
|
||||
assert cost == pytest.approx(0.6) # 2 * (0.1 + 0.2)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
||||
assert result.cost == pytest.approx(0.6) # 2 * (0.1 + 0.2)
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
|
||||
30,
|
||||
15,
|
||||
45,
|
||||
)
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 0
|
||||
|
||||
|
||||
def test_vertex_cost_skips_none_response_body(monkeypatch):
|
||||
|
|
@ -627,10 +702,12 @@ def test_vertex_cost_skips_none_response_body(monkeypatch):
|
|||
},
|
||||
]
|
||||
|
||||
cost, usage = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
result = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
|
||||
assert cost == pytest.approx(1.0) # only one line costed
|
||||
assert usage.total_tokens == 10
|
||||
assert result.cost == pytest.approx(1.0) # only one line costed
|
||||
assert result.usage.total_tokens == 10
|
||||
assert result.successful_requests == 1
|
||||
assert result.failed_requests == 1
|
||||
|
||||
|
||||
def test_vertex_usage_total_token_fallback(monkeypatch):
|
||||
|
|
@ -640,8 +717,8 @@ def test_vertex_usage_total_token_fallback(monkeypatch):
|
|||
monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.0, 0.0))
|
||||
responses = [{"response": {"usageMetadata": {"promptTokenCount": 8, "candidatesTokenCount": 4}}}]
|
||||
|
||||
_, usage = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
assert usage.total_tokens == 12
|
||||
result = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
assert result.usage.total_tokens == 12
|
||||
|
||||
|
||||
def test_vertex_cost_error_in_line_is_swallowed(monkeypatch):
|
||||
|
|
@ -664,9 +741,9 @@ def test_vertex_cost_error_in_line_is_swallowed(monkeypatch):
|
|||
}
|
||||
]
|
||||
|
||||
cost, usage = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
assert cost == 0.0
|
||||
assert usage.total_tokens == 10
|
||||
result = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-x")
|
||||
assert result.cost == 0.0
|
||||
assert result.usage.total_tokens == 10
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
@ -679,13 +756,11 @@ async def test_calculate_batch_cost_and_usage_orchestration(monkeypatch):
|
|||
rows = [_success_row(model="gpt-4o", usage=_usage(10, 5))]
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 2.5)
|
||||
|
||||
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=rows, custom_llm_provider="openai"
|
||||
)
|
||||
result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=rows, custom_llm_provider="openai")
|
||||
|
||||
assert cost == 2.5
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (10, 5, 15)
|
||||
assert models == ["gpt-4o"]
|
||||
assert result.cost == 2.5
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (10, 5, 15)
|
||||
assert result.models == ["gpt-4o"]
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
@ -940,7 +1015,7 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk
|
|||
|
||||
monkeypatch.setattr(files_main, "afile_content", fake_afile_content)
|
||||
|
||||
cost, usage, models = await bu._handle_completed_batch(
|
||||
result = await bu._handle_completed_batch(
|
||||
_batch("gs://litellm-bucket/output/predictions.jsonl"),
|
||||
custom_llm_provider="vertex_ai",
|
||||
litellm_params={"vertex_project": "proj-1", "vertex_location": "us-central1"},
|
||||
|
|
@ -952,10 +1027,12 @@ async def test_handle_completed_vertex_batch_computes_cost_usage_and_models(monk
|
|||
|
||||
assert batch_input < pricing["input_cost_per_token"]
|
||||
assert batch_output < pricing["output_cost_per_token"]
|
||||
assert cost > 0
|
||||
assert cost == pytest.approx(30 * batch_input + 15 * batch_output)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (30, 15, 45)
|
||||
assert models == ["gemini-3.6-flash", "gemini-3.6-flash"]
|
||||
assert result.cost > 0
|
||||
assert result.cost == pytest.approx(30 * batch_input + 15 * batch_output)
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (30, 15, 45)
|
||||
assert result.models == ["gemini-3.6-flash", "gemini-3.6-flash"]
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1033,11 +1110,121 @@ async def test_handle_completed_batch_orchestration(monkeypatch):
|
|||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 3.3)
|
||||
|
||||
cost, usage, models = await bu._handle_completed_batch(_batch("of"), custom_llm_provider="openai")
|
||||
result = await bu._handle_completed_batch(_batch("of"), custom_llm_provider="openai")
|
||||
|
||||
assert cost == 3.3
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (10, 5, 15)
|
||||
assert models == ["gpt-4o"]
|
||||
assert result.cost == 3.3
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (10, 5, 15)
|
||||
assert result.models == ["gpt-4o"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_completed_batch_counts_error_file_failures(monkeypatch):
|
||||
"""Regression test: OpenAI writes per-request failures (e.g. a rejected param)
|
||||
to a separate error_file_id, never into the output file - so failed_requests
|
||||
must include them or it silently undercounts real batch failures."""
|
||||
from litellm.types.llms.openai import Batch
|
||||
|
||||
rows = [_success_row(model="gpt-5-mini", usage=_usage(24, 107))]
|
||||
error_rows = [
|
||||
{
|
||||
"id": "batch_req_err1",
|
||||
"custom_id": "req-2-bad",
|
||||
"response": {"status_code": 400, "body": {"error": {"message": "Invalid 'temperature'"}}},
|
||||
"error": None,
|
||||
}
|
||||
]
|
||||
|
||||
async def fake_fetch(batch, custom_llm_provider, litellm_params=None):
|
||||
return _vertex_jsonl(rows)
|
||||
|
||||
async def fake_afile_content(**kw):
|
||||
return type("R", (), {"content": _vertex_jsonl(error_rows)})()
|
||||
|
||||
import litellm.files.main as files_main
|
||||
|
||||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
|
||||
monkeypatch.setattr(files_main, "afile_content", fake_afile_content)
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
|
||||
batch = Batch(
|
||||
id="b",
|
||||
completion_window="24h",
|
||||
created_at=1,
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id="f",
|
||||
object="batch",
|
||||
status="completed",
|
||||
output_file_id="of",
|
||||
error_file_id="ef",
|
||||
)
|
||||
|
||||
result = await bu._handle_completed_batch(batch, custom_llm_provider="openai")
|
||||
|
||||
assert result.successful_requests == 1
|
||||
assert result.failed_requests == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_completed_batch_decodes_model_encoded_error_file_id(monkeypatch):
|
||||
"""A model-encoded error file id must be decoded to the raw provider id before
|
||||
the fetch, exactly like the output file id. Sending the encoded id straight to
|
||||
the provider 404s, and the swallowed fetch failure silently reports 0 failures."""
|
||||
import base64
|
||||
|
||||
from litellm.types.llms.openai import Batch
|
||||
|
||||
provider_error_file_id = "file-real-error-id"
|
||||
encoded_error_file_id = "file-" + base64.urlsafe_b64encode(
|
||||
f"litellm:{provider_error_file_id};model,model-abc".encode()
|
||||
).decode().rstrip("=")
|
||||
|
||||
requested_file_ids = []
|
||||
|
||||
async def fake_fetch(batch, custom_llm_provider, litellm_params=None):
|
||||
return _vertex_jsonl([_success_row(model="gpt-4o", usage=_usage(10, 5))])
|
||||
|
||||
async def fake_afile_content(**kw):
|
||||
requested_file_ids.append(kw["file_id"])
|
||||
return type("R", (), {"content": _vertex_jsonl([{"custom_id": "bad-1"}])})()
|
||||
|
||||
import litellm.files.main as files_main
|
||||
|
||||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
|
||||
monkeypatch.setattr(files_main, "afile_content", fake_afile_content)
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
|
||||
batch = Batch(
|
||||
id="b",
|
||||
completion_window="24h",
|
||||
created_at=1,
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id="f",
|
||||
object="batch",
|
||||
status="completed",
|
||||
output_file_id="of",
|
||||
error_file_id=encoded_error_file_id,
|
||||
)
|
||||
|
||||
result = await bu._handle_completed_batch(batch, custom_llm_provider="openai")
|
||||
|
||||
assert requested_file_ids == [provider_error_file_id]
|
||||
assert result.failed_requests == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_completed_batch_no_error_file_id_reports_zero_error_failures(monkeypatch):
|
||||
rows = [_success_row(model="gpt-4o", usage=_usage(10, 5))]
|
||||
|
||||
async def fake_fetch(batch, custom_llm_provider, litellm_params=None):
|
||||
return _vertex_jsonl(rows)
|
||||
|
||||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
|
||||
monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.0)
|
||||
|
||||
result = await bu._handle_completed_batch(_batch("of"), custom_llm_provider="openai")
|
||||
|
||||
assert result.successful_requests == 1
|
||||
assert result.failed_requests == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1054,11 +1241,13 @@ async def test_handle_completed_batch_no_output_file_is_zero(monkeypatch):
|
|||
|
||||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", _must_not_fetch)
|
||||
|
||||
cost, usage, models = await bu._handle_completed_batch(_batch(None), custom_llm_provider="openai")
|
||||
result = await bu._handle_completed_batch(_batch(None), custom_llm_provider="openai")
|
||||
|
||||
assert cost == 0.0
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0)
|
||||
assert models == []
|
||||
assert result.cost == 0.0
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (0, 0, 0)
|
||||
assert result.models == []
|
||||
assert result.successful_requests == 0
|
||||
assert result.failed_requests == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1075,19 +1264,25 @@ async def test_handle_completed_batch_vertex_disable_transform_path(monkeypatch)
|
|||
def fake_vertex_calc(content, model):
|
||||
seen["content"] = content
|
||||
seen["model"] = model
|
||||
return 7.7, Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3)
|
||||
return bu.BatchCostUsageResult(
|
||||
cost=7.7,
|
||||
usage=Usage(prompt_tokens=1, completion_tokens=2, total_tokens=3),
|
||||
models=["gemini-x"],
|
||||
successful_requests=1,
|
||||
failed_requests=0,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(bu, "calculate_vertex_ai_batch_cost_and_usage", fake_vertex_calc)
|
||||
|
||||
cost, usage, models = await bu._handle_completed_batch(
|
||||
result = await bu._handle_completed_batch(
|
||||
_batch("gs://litellm-bucket/output/predictions.jsonl"),
|
||||
custom_llm_provider="vertex_ai",
|
||||
model_name="gemini-x",
|
||||
)
|
||||
|
||||
assert cost == 7.7
|
||||
assert usage.total_tokens == 3
|
||||
assert models == ["gemini-x"]
|
||||
assert result.cost == 7.7
|
||||
assert result.usage.total_tokens == 3
|
||||
assert result.models == ["gemini-x"]
|
||||
assert seen["content"] == raw_rows
|
||||
assert seen["model"] == "gemini-x"
|
||||
|
||||
|
|
@ -1189,14 +1384,14 @@ def test_bedrock_cost_uses_deployment_model_name():
|
|||
"recordId": "1",
|
||||
"modelOutput": {"model": "claude-sonnet-4-6", "usage": {"input_tokens": 13, "output_tokens": 5}},
|
||||
}
|
||||
cost, _, models = bu._aggregate_batch_cost_usage_models(
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[row],
|
||||
custom_llm_provider="bedrock",
|
||||
model_name="us.anthropic.claude-sonnet-4-6",
|
||||
model_info={},
|
||||
)
|
||||
assert cost > 0
|
||||
assert models == ["us.anthropic.claude-sonnet-4-6"]
|
||||
assert result.cost > 0
|
||||
assert result.models == ["us.anthropic.claude-sonnet-4-6"]
|
||||
|
||||
|
||||
def test_anthropic_total_usage_sums_succeeded_only(monkeypatch):
|
||||
|
|
@ -1208,8 +1403,10 @@ def test_anthropic_total_usage_sums_succeeded_only(monkeypatch):
|
|||
_anthropic_errored_row(),
|
||||
_anthropic_succeeded_row(usage=_anthropic_usage(20, 10, cache_read=100)),
|
||||
]
|
||||
_, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic")
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (130, 15, 145)
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic")
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (130, 15, 145)
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 1
|
||||
|
||||
|
||||
def test_anthropic_total_usage_aggregates_cache_token_details(monkeypatch):
|
||||
|
|
@ -1221,11 +1418,11 @@ def test_anthropic_total_usage_aggregates_cache_token_details(monkeypatch):
|
|||
_anthropic_errored_row(),
|
||||
_anthropic_succeeded_row(usage=_anthropic_usage(50, 20, cache_creation=300, cache_read=700)),
|
||||
]
|
||||
_, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic")
|
||||
assert usage.prompt_tokens_details.cached_tokens == 8700
|
||||
assert usage.prompt_tokens_details.cache_creation_tokens == 2300
|
||||
assert usage.cache_read_input_tokens == 8700
|
||||
assert usage.cache_creation_input_tokens == 2300
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic")
|
||||
assert result.usage.prompt_tokens_details.cached_tokens == 8700
|
||||
assert result.usage.prompt_tokens_details.cache_creation_tokens == 2300
|
||||
assert result.usage.cache_read_input_tokens == 8700
|
||||
assert result.usage.cache_creation_input_tokens == 2300
|
||||
|
||||
|
||||
def test_total_usage_without_cache_tokens_has_no_prompt_details(monkeypatch):
|
||||
|
|
@ -1236,9 +1433,9 @@ def test_total_usage_without_cache_tokens_has_no_prompt_details(monkeypatch):
|
|||
"response": {"status_code": 200, "body": {"model": "gpt-5.2", "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}},
|
||||
}
|
||||
]
|
||||
_, usage, _ = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (10, 5, 15)
|
||||
assert usage.prompt_tokens_details is None
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (10, 5, 15)
|
||||
assert result.usage.prompt_tokens_details is None
|
||||
|
||||
|
||||
def test_anthropic_cost_applies_batch_discount_and_cache_pricing():
|
||||
|
|
@ -1249,14 +1446,14 @@ def test_anthropic_cost_applies_batch_discount_and_cache_pricing():
|
|||
_anthropic_errored_row(),
|
||||
]
|
||||
|
||||
total, _, _ = bu._aggregate_batch_cost_usage_models(
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=rows,
|
||||
custom_llm_provider="anthropic",
|
||||
model_info=_ANTHROPIC_MODEL_INFO, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
expected_half_price = (1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3.75e-6 + 200 * 15e-6) / 2
|
||||
assert total == pytest.approx(expected_half_price)
|
||||
assert result.cost == pytest.approx(expected_half_price)
|
||||
|
||||
|
||||
def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatch):
|
||||
|
|
@ -1275,11 +1472,9 @@ def test_anthropic_cost_without_model_info_uses_batch_cost_calculator(monkeypatc
|
|||
lambda **kw: pytest.fail("anthropic rows must not go through completion_cost"),
|
||||
)
|
||||
|
||||
total, _, _ = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic"
|
||||
)
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=[_anthropic_succeeded_row()], custom_llm_provider="anthropic")
|
||||
|
||||
assert total == pytest.approx(0.3)
|
||||
assert result.cost == pytest.approx(0.3)
|
||||
assert seen[0]["model"] == "claude-sonnet-4-5-20250929"
|
||||
assert seen[0]["custom_llm_provider"] == "anthropic"
|
||||
assert seen[0]["usage"].prompt_tokens == 10
|
||||
|
|
@ -1293,8 +1488,8 @@ def test_anthropic_batch_models_collected_from_succeeded_rows(monkeypatch):
|
|||
_anthropic_succeeded_row(model="claude-sonnet-4-5-20250929"),
|
||||
_anthropic_errored_row(),
|
||||
]
|
||||
_, _, models = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic")
|
||||
assert models == ["claude-sonnet-4-5-20250929"]
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="anthropic")
|
||||
assert result.models == ["claude-sonnet-4-5-20250929"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1304,16 +1499,16 @@ async def test_calculate_batch_cost_and_usage_anthropic_end_to_end():
|
|||
_anthropic_errored_row(),
|
||||
]
|
||||
|
||||
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
||||
result = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=rows,
|
||||
custom_llm_provider="anthropic",
|
||||
model_name="claude-sonnet-4-5",
|
||||
model_info=_ANTHROPIC_MODEL_INFO, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(1000 * 3e-6 / 2 + 8000 * 3e-7 / 2 + 2000 * 3.75e-6 / 2 + 200 * 15e-6 / 2)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (11000, 200, 11200)
|
||||
assert models == ["claude-sonnet-4-5"]
|
||||
assert result.cost == pytest.approx(1000 * 3e-6 / 2 + 8000 * 3e-7 / 2 + 2000 * 3.75e-6 / 2 + 200 * 15e-6 / 2)
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (11000, 200, 11200)
|
||||
assert result.models == ["claude-sonnet-4-5"]
|
||||
|
||||
|
||||
def test_extract_credentials_forwards_the_trusted_model_credential_snapshot():
|
||||
|
|
@ -1421,24 +1616,24 @@ async def test_handle_completed_bedrock_batch_prices_from_deployment_model(monke
|
|||
|
||||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
|
||||
|
||||
cost, usage, _ = await bu._handle_completed_batch(
|
||||
result = await bu._handle_completed_batch(
|
||||
_batch("of"),
|
||||
custom_llm_provider="bedrock",
|
||||
model_name="bedrock/global.anthropic.claude-sonnet-4-6",
|
||||
)
|
||||
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (1800, 1000, 2800)
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (1800, 1000, 2800)
|
||||
# 3e-06 / 1.5e-05 on-demand, halved for batch.
|
||||
assert cost == pytest.approx(1800 * 3e-06 / 2 + 1000 * 1.5e-05 / 2)
|
||||
assert result.cost == pytest.approx(1800 * 3e-06 / 2 + 1000 * 1.5e-05 / 2)
|
||||
|
||||
# The response model alone cannot price a bedrock batch: this is the $0 bug.
|
||||
zero_cost, zero_usage, _ = await bu._handle_completed_batch(
|
||||
zero_result = await bu._handle_completed_batch(
|
||||
_batch("of"),
|
||||
custom_llm_provider="bedrock",
|
||||
model_name=None,
|
||||
)
|
||||
assert zero_cost == 0.0
|
||||
assert zero_usage.total_tokens == 2800
|
||||
assert zero_result.cost == 0.0
|
||||
assert zero_result.usage.total_tokens == 2800
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1451,7 +1646,7 @@ async def test_handle_completed_batch_honors_deployment_pricing(monkeypatch) ->
|
|||
|
||||
monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
|
||||
|
||||
free_cost, _, _ = await bu._handle_completed_batch(
|
||||
free_result = await bu._handle_completed_batch(
|
||||
_batch("of"),
|
||||
custom_llm_provider="vertex_ai",
|
||||
model_name="vertex_ai/gemini-2.5-flash",
|
||||
|
|
@ -1462,15 +1657,15 @@ async def test_handle_completed_batch_honors_deployment_pricing(monkeypatch) ->
|
|||
"output_cost_per_token_batches": 0.0,
|
||||
},
|
||||
)
|
||||
assert free_cost == 0.0
|
||||
assert free_result.cost == 0.0
|
||||
|
||||
billed_cost, _, _ = await bu._handle_completed_batch(
|
||||
billed_result = await bu._handle_completed_batch(
|
||||
_batch("of"),
|
||||
custom_llm_provider="vertex_ai",
|
||||
model_name="vertex_ai/gemini-2.5-flash",
|
||||
model_info=None,
|
||||
)
|
||||
assert billed_cost > 0.0
|
||||
assert billed_result.cost > 0.0
|
||||
|
||||
|
||||
# =========================================================================== #
|
||||
|
|
|
|||
|
|
@ -71,24 +71,24 @@ async def test_responses_batch_reconciles_to_real_tokens_and_spend(local_model_c
|
|||
input_tokens = 33
|
||||
output_tokens = 57
|
||||
|
||||
cost, usage, models = await bu.calculate_batch_cost_and_usage(
|
||||
result = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[_responses_line(input_tokens, output_tokens)],
|
||||
custom_llm_provider="openai",
|
||||
model_name=MODEL,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
input_tokens + output_tokens,
|
||||
)
|
||||
assert models == [MODEL]
|
||||
assert cost == pytest.approx(
|
||||
assert result.models == [MODEL]
|
||||
assert result.cost == pytest.approx(
|
||||
input_tokens * model_info["input_cost_per_token_batches"]
|
||||
+ output_tokens * model_info["output_cost_per_token_batches"]
|
||||
)
|
||||
assert cost > 0.0
|
||||
assert result.cost > 0.0
|
||||
|
||||
|
||||
async def test_mixed_shape_batch_output_sums_across_both_line_shapes(local_model_cost_map):
|
||||
|
|
@ -96,15 +96,15 @@ async def test_mixed_shape_batch_output_sums_across_both_line_shapes(local_model
|
|||
batch's declared endpoint rather than each line's shape would miss this."""
|
||||
model_info = litellm.get_model_info(model=MODEL, custom_llm_provider="openai")
|
||||
|
||||
cost, usage, _ = await bu.calculate_batch_cost_and_usage(
|
||||
result = await bu.calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=[_responses_line(100, 50), _chat_line(33, 57)],
|
||||
custom_llm_provider="openai",
|
||||
model_name=MODEL,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (133, 107, 240)
|
||||
assert cost == pytest.approx(
|
||||
assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (133, 107, 240)
|
||||
assert result.cost == pytest.approx(
|
||||
133 * model_info["input_cost_per_token_batches"] + 107 * model_info["output_cost_per_token_batches"]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -580,9 +580,17 @@ class TestRetrieveBatchCostPassesModelIdentity:
|
|||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_handle_completed_batch(**kwargs: object) -> tuple[float, Usage, list[str]]:
|
||||
from litellm.batches.batch_utils import BatchCostUsageResult
|
||||
|
||||
async def fake_handle_completed_batch(**kwargs: object) -> BatchCostUsageResult:
|
||||
captured.update(kwargs)
|
||||
return 1.25, Usage(prompt_tokens=1800, completion_tokens=1000, total_tokens=2800), ["m"]
|
||||
return BatchCostUsageResult(
|
||||
cost=1.25,
|
||||
usage=Usage(prompt_tokens=1800, completion_tokens=1000, total_tokens=2800),
|
||||
models=["m"],
|
||||
successful_requests=1,
|
||||
failed_requests=0,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(logging_module, "_handle_completed_batch", fake_handle_completed_batch)
|
||||
|
||||
|
|
|
|||
|
|
@ -478,14 +478,14 @@ class TestVertexAIBatchPassthroughHandler:
|
|||
}
|
||||
]
|
||||
|
||||
total_cost, usage = calculate_vertex_ai_batch_cost_and_usage(
|
||||
result = calculate_vertex_ai_batch_cost_and_usage(
|
||||
vertex_ai_batch_responses, model_name="gemini-2.0-flash-001"
|
||||
)
|
||||
|
||||
assert usage.total_tokens == 15
|
||||
assert usage.prompt_tokens == 10
|
||||
assert usage.completion_tokens == 5
|
||||
assert total_cost > 0, "batch_cost_calculator should return a non-zero cost"
|
||||
assert result.usage.total_tokens == 15
|
||||
assert result.usage.prompt_tokens == 10
|
||||
assert result.usage.completion_tokens == 5
|
||||
assert result.cost > 0, "batch_cost_calculator should return a non-zero cost"
|
||||
|
||||
def test_batch_response_transformation(self):
|
||||
"""Test transformation of Vertex AI batch responses to OpenAI format"""
|
||||
|
|
@ -664,14 +664,14 @@ class TestVertexAIBatchCostCalculation:
|
|||
},
|
||||
]
|
||||
|
||||
total_cost, usage = calculate_vertex_ai_batch_cost_and_usage(
|
||||
result = calculate_vertex_ai_batch_cost_and_usage(
|
||||
responses, model_name="gemini-2.0-flash-001"
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 18
|
||||
assert usage.completion_tokens == 8
|
||||
assert usage.total_tokens == 26
|
||||
assert total_cost > 0, "batch_cost_calculator should return a non-zero cost"
|
||||
assert result.usage.prompt_tokens == 18
|
||||
assert result.usage.completion_tokens == 8
|
||||
assert result.usage.total_tokens == 26
|
||||
assert result.cost > 0, "batch_cost_calculator should return a non-zero cost"
|
||||
|
||||
def test_should_skip_responses_with_null_response_body(self):
|
||||
"""Failed lines (response: None) are skipped without error."""
|
||||
|
|
@ -699,27 +699,29 @@ class TestVertexAIBatchCostCalculation:
|
|||
},
|
||||
]
|
||||
|
||||
total_cost, usage = calculate_vertex_ai_batch_cost_and_usage(
|
||||
result = calculate_vertex_ai_batch_cost_and_usage(
|
||||
responses, model_name="gemini-2.0-flash-001"
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 18
|
||||
assert usage.completion_tokens == 8
|
||||
assert usage.total_tokens == 26
|
||||
assert total_cost > 0
|
||||
assert result.usage.prompt_tokens == 18
|
||||
assert result.usage.completion_tokens == 8
|
||||
assert result.usage.total_tokens == 26
|
||||
assert result.cost > 0
|
||||
assert result.successful_requests == 2
|
||||
assert result.failed_requests == 1
|
||||
|
||||
def test_should_return_zeros_for_empty_response_list(self):
|
||||
"""Empty input → zero cost and zero usage."""
|
||||
from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage
|
||||
|
||||
total_cost, usage = calculate_vertex_ai_batch_cost_and_usage(
|
||||
result = calculate_vertex_ai_batch_cost_and_usage(
|
||||
[], model_name="gemini-2.0-flash-001"
|
||||
)
|
||||
|
||||
assert total_cost == 0.0
|
||||
assert usage.total_tokens == 0
|
||||
assert usage.prompt_tokens == 0
|
||||
assert usage.completion_tokens == 0
|
||||
assert result.cost == 0.0
|
||||
assert result.usage.total_tokens == 0
|
||||
assert result.usage.prompt_tokens == 0
|
||||
assert result.usage.completion_tokens == 0
|
||||
|
||||
def test_should_handle_missing_usage_metadata_gracefully(self):
|
||||
"""Response without usageMetadata → 0 tokens, 0 cost for that line."""
|
||||
|
|
@ -729,13 +731,13 @@ class TestVertexAIBatchCostCalculation:
|
|||
{"response": {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}},
|
||||
]
|
||||
|
||||
total_cost, usage = calculate_vertex_ai_batch_cost_and_usage(
|
||||
result = calculate_vertex_ai_batch_cost_and_usage(
|
||||
responses, model_name="gemini-2.0-flash-001"
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 0
|
||||
assert usage.completion_tokens == 0
|
||||
assert usage.total_tokens == 0
|
||||
assert result.usage.prompt_tokens == 0
|
||||
assert result.usage.completion_tokens == 0
|
||||
assert result.usage.total_tokens == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_shaped_output_records_nonzero_cost_and_usage(self):
|
||||
|
|
@ -813,7 +815,7 @@ class TestVertexAIBatchCostCalculation:
|
|||
try:
|
||||
litellm.disable_vertex_batch_output_transformation = False
|
||||
|
||||
cost, usage, _ = await calculate_batch_cost_and_usage(
|
||||
result = await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=openai_shaped_responses,
|
||||
custom_llm_provider="vertex_ai",
|
||||
model_name="gemini-2.0-flash-001",
|
||||
|
|
@ -822,17 +824,17 @@ class TestVertexAIBatchCostCalculation:
|
|||
litellm.disable_vertex_batch_output_transformation = original_flag
|
||||
|
||||
assert (
|
||||
usage.prompt_tokens == 18
|
||||
), f"expected 18 prompt tokens, got {usage.prompt_tokens}"
|
||||
result.usage.prompt_tokens == 18
|
||||
), f"expected 18 prompt tokens, got {result.usage.prompt_tokens}"
|
||||
assert (
|
||||
usage.completion_tokens == 8
|
||||
), f"expected 8 completion tokens, got {usage.completion_tokens}"
|
||||
result.usage.completion_tokens == 8
|
||||
), f"expected 8 completion tokens, got {result.usage.completion_tokens}"
|
||||
assert (
|
||||
usage.total_tokens == 26
|
||||
), f"expected 26 total tokens, got {usage.total_tokens}"
|
||||
result.usage.total_tokens == 26
|
||||
), f"expected 26 total tokens, got {result.usage.total_tokens}"
|
||||
assert (
|
||||
cost > 0
|
||||
), f"expected non-zero cost for completed Vertex batch, got {cost}"
|
||||
result.cost > 0
|
||||
), f"expected non-zero cost for completed Vertex batch, got {result.cost}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_vertex_output_still_works_when_transformation_disabled(self):
|
||||
|
|
@ -865,7 +867,7 @@ class TestVertexAIBatchCostCalculation:
|
|||
try:
|
||||
litellm.disable_vertex_batch_output_transformation = True
|
||||
|
||||
cost, usage, _ = await calculate_batch_cost_and_usage(
|
||||
result = await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=raw_vertex_responses,
|
||||
custom_llm_provider="vertex_ai",
|
||||
model_name="gemini-2.0-flash-001",
|
||||
|
|
@ -873,7 +875,7 @@ class TestVertexAIBatchCostCalculation:
|
|||
finally:
|
||||
litellm.disable_vertex_batch_output_transformation = original_flag
|
||||
|
||||
assert usage.prompt_tokens == 10
|
||||
assert usage.completion_tokens == 5
|
||||
assert usage.total_tokens == 15
|
||||
assert cost > 0, "raw Vertex shape should also produce non-zero cost"
|
||||
assert result.usage.prompt_tokens == 10
|
||||
assert result.usage.completion_tokens == 5
|
||||
assert result.usage.total_tokens == 15
|
||||
assert result.cost > 0, "raw Vertex shape should also produce non-zero cost"
|
||||
|
|
|
|||
|
|
@ -2865,7 +2865,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
@ -2961,7 +2961,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
@ -3055,7 +3055,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue