mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
fix(cost): resolve a missing 1h cache write rate after off-peak pricing
The one-hour cache write fallback now takes the applied cache write rate, so an off-peak write price carries into it instead of the input rate The cost estimate test for a cost-map model without cache prices now expects writes at the input rate, which is what the proxy bills The recording logger in the deferred guardrail test types its callback parameters
This commit is contained in:
parent
f8c2539ba7
commit
8573241c49
4 changed files with 35 additions and 19 deletions
|
|
@ -484,11 +484,12 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
|
|||
def _apply_off_peak_to_base_costs(
|
||||
model_info: ModelInfo,
|
||||
current_time: datetime | None,
|
||||
base_costs: tuple[float, float, float, float, float],
|
||||
base_costs: tuple[float, float, float, float | None, float],
|
||||
) -> tuple[float, float, float, float, float]:
|
||||
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
|
||||
produced them. The one-hour cache-creation rate passes through untouched, since
|
||||
off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
|
||||
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
|
||||
present one passes through untouched and an absent one resolves to the applied
|
||||
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
|
||||
"""
|
||||
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
|
||||
rates: Final = apply_off_peak_pricing(
|
||||
|
|
@ -506,7 +507,7 @@ def _apply_off_peak_to_base_costs(
|
|||
rates.input_rate,
|
||||
rates.output_rate,
|
||||
rates.cache_creation_rate,
|
||||
cache_creation_above_1hr,
|
||||
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
|
||||
rates.cache_read_rate,
|
||||
)
|
||||
|
||||
|
|
@ -535,7 +536,7 @@ def _get_token_base_cost(
|
|||
An absent cache-creation rate always resolves to the resolved input rate, the way the
|
||||
tiered table and custom deployment pricing already do, since a provider that publishes
|
||||
no write price bills cache writes as ordinary input. An absent 1h write rate resolves
|
||||
to the cache-creation rate. An explicit 0.0 stays a real price for both.
|
||||
to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both.
|
||||
|
||||
Returns:
|
||||
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
|
||||
|
|
@ -667,9 +668,6 @@ def _get_token_base_cost(
|
|||
resolved_cache_creation_cost: Final = (
|
||||
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
|
||||
)
|
||||
resolved_cache_creation_cost_above_1hr: Final = (
|
||||
resolved_cache_creation_cost if cache_creation_cost_above_1hr is None else cache_creation_cost_above_1hr
|
||||
)
|
||||
|
||||
return _apply_off_peak_to_base_costs(
|
||||
model_info,
|
||||
|
|
@ -678,7 +676,7 @@ def _get_token_base_cost(
|
|||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
resolved_cache_creation_cost,
|
||||
resolved_cache_creation_cost_above_1hr,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5476,6 +5476,19 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a
|
|||
1e-7,
|
||||
id="no-write-price-uses-the-off-peak-input-rate",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "00:00-23:59",
|
||||
"input_cost_per_token": 1e-7,
|
||||
"cache_creation_input_token_cost": 3e-7,
|
||||
}
|
||||
},
|
||||
datetime(2026, 9, 14, 12, tzinfo=timezone.utc),
|
||||
3e-7,
|
||||
3e-7,
|
||||
id="no-1h-price-uses-the-off-peak-write-price",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path(
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ Streaming: CSW.__anext__ stores args on logging_obj at stream end.
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime
|
||||
from typing import Any, Final
|
||||
from typing import Any, Final, cast
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -76,8 +76,10 @@ class _RecordingLogger(CustomLogger):
|
|||
super().__init__()
|
||||
self.standard_logging_object: StandardLoggingPayload | None = None
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
self.standard_logging_object = kwargs["standard_logging_object"]
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
self.standard_logging_object = cast(StandardLoggingPayload, kwargs["standard_logging_object"])
|
||||
|
||||
|
||||
class PostCallGuardrail(CustomGuardrail):
|
||||
|
|
|
|||
|
|
@ -975,8 +975,9 @@ class TestEstimateCostCacheAndReasoningTokens:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch):
|
||||
"""The cost calculator bills cache tokens of a cost-map model without cache prices at zero
|
||||
and its reasoning tokens at the output rate. The estimate reports those effective rates."""
|
||||
"""The cost calculator bills cache reads of a cost-map model without cache prices at zero,
|
||||
its cache writes at the input rate, and its reasoning tokens at the output rate. The estimate
|
||||
reports those effective rates."""
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
A_MAPPED_MODEL,
|
||||
|
|
@ -986,12 +987,14 @@ class TestEstimateCostCacheAndReasoningTokens:
|
|||
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL)
|
||||
|
||||
assert response.cache_read_cost_per_request == 0.0
|
||||
assert response.cache_creation_cost_per_request == 0.0
|
||||
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 5e-6)
|
||||
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6)
|
||||
assert response.input_cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6)
|
||||
assert response.cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6)
|
||||
assert response.input_cost_per_request == pytest.approx((TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6)
|
||||
assert response.cost_per_request == pytest.approx(
|
||||
(TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6 + OUTPUT_TOKENS * 6e-6
|
||||
)
|
||||
assert response.cache_read_input_token_cost == 0.0
|
||||
assert response.cache_creation_input_token_cost == 0.0
|
||||
assert response.cache_creation_input_token_cost == pytest.approx(5e-6)
|
||||
assert response.output_cost_per_reasoning_token == pytest.approx(6e-6)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue