fix(cost): resolve a missing 1h cache write rate after off-peak pricing

The one-hour cache write fallback now takes the applied cache write rate, so an off-peak write price carries into it instead of the input rate

The cost estimate test for a cost-map model without cache prices now expects writes at the input rate, which is what the proxy bills

The recording logger in the deferred guardrail test types its callback parameters
This commit is contained in:
mateo-berri 2026-09-14 21:46:33 -07:00
parent f8c2539ba7
commit 8573241c49
4 changed files with 35 additions and 19 deletions

View file

@ -484,11 +484,12 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
def _apply_off_peak_to_base_costs(
model_info: ModelInfo,
current_time: datetime | None,
base_costs: tuple[float, float, float, float, float],
base_costs: tuple[float, float, float, float | None, float],
) -> tuple[float, float, float, float, float]:
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
produced them. The one-hour cache-creation rate passes through untouched, since
off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
present one passes through untouched and an absent one resolves to the applied
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
"""
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
rates: Final = apply_off_peak_pricing(
@ -506,7 +507,7 @@ def _apply_off_peak_to_base_costs(
rates.input_rate,
rates.output_rate,
rates.cache_creation_rate,
cache_creation_above_1hr,
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
rates.cache_read_rate,
)
@ -535,7 +536,7 @@ def _get_token_base_cost(
An absent cache-creation rate always resolves to the resolved input rate, the way the
tiered table and custom deployment pricing already do, since a provider that publishes
no write price bills cache writes as ordinary input. An absent 1h write rate resolves
to the cache-creation rate. An explicit 0.0 stays a real price for both.
to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
@ -667,9 +668,6 @@ def _get_token_base_cost(
resolved_cache_creation_cost: Final = (
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
)
resolved_cache_creation_cost_above_1hr: Final = (
resolved_cache_creation_cost if cache_creation_cost_above_1hr is None else cache_creation_cost_above_1hr
)
return _apply_off_peak_to_base_costs(
model_info,
@ -678,7 +676,7 @@ def _get_token_base_cost(
prompt_base_cost,
completion_base_cost,
resolved_cache_creation_cost,
resolved_cache_creation_cost_above_1hr,
cache_creation_cost_above_1hr,
cache_read_cost,
),
)

View file

@ -5476,6 +5476,19 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a
1e-7,
id="no-write-price-uses-the-off-peak-input-rate",
),
pytest.param(
{
"off_peak_pricing": {
"hours_utc": "00:00-23:59",
"input_cost_per_token": 1e-7,
"cache_creation_input_token_cost": 3e-7,
}
},
datetime(2026, 9, 14, 12, tzinfo=timezone.utc),
3e-7,
3e-7,
id="no-1h-price-uses-the-off-peak-write-price",
),
),
)
def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path(

View file

@ -16,9 +16,9 @@ Streaming: CSW.__anext__ stores args on logging_obj at stream end.
import asyncio
import logging
from collections.abc import Callable
from collections.abc import Callable, Mapping
from datetime import datetime
from typing import Any, Final
from typing import Any, Final, cast
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@ -76,8 +76,10 @@ class _RecordingLogger(CustomLogger):
super().__init__()
self.standard_logging_object: StandardLoggingPayload | None = None
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
self.standard_logging_object = kwargs["standard_logging_object"]
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
self.standard_logging_object = cast(StandardLoggingPayload, kwargs["standard_logging_object"])
class PostCallGuardrail(CustomGuardrail):

View file

@ -975,8 +975,9 @@ class TestEstimateCostCacheAndReasoningTokens:
@pytest.mark.asyncio
async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch):
"""The cost calculator bills cache tokens of a cost-map model without cache prices at zero
and its reasoning tokens at the output rate. The estimate reports those effective rates."""
"""The cost calculator bills cache reads of a cost-map model without cache prices at zero,
its cache writes at the input rate, and its reasoning tokens at the output rate. The estimate
reports those effective rates."""
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
@ -986,12 +987,14 @@ class TestEstimateCostCacheAndReasoningTokens:
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL)
assert response.cache_read_cost_per_request == 0.0
assert response.cache_creation_cost_per_request == 0.0
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 5e-6)
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6)
assert response.input_cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6)
assert response.cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6)
assert response.input_cost_per_request == pytest.approx((TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6)
assert response.cost_per_request == pytest.approx(
(TEXT_INPUT_TOKENS + CACHE_CREATION_TOKENS) * 5e-6 + OUTPUT_TOKENS * 6e-6
)
assert response.cache_read_input_token_cost == 0.0
assert response.cache_creation_input_token_cost == 0.0
assert response.cache_creation_input_token_cost == pytest.approx(5e-6)
assert response.output_cost_per_reasoning_token == pytest.approx(6e-6)
@pytest.mark.asyncio