From e65e3d0e2b89becc8fb55268ff18a8141e9ddf4b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:45:17 -0700 Subject: [PATCH] fix(cost): bill fireworks cached tokens at the off-peak input rate when no cache-read rate exists --- litellm/llms/fireworks_ai/cost_calculator.py | 11 +++++---- .../test_fireworks_ai_cost_calculator.py | 23 +++++++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index df47d3546ca..3843bad6d8f 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -2,6 +2,7 @@ For calculating cost of fireworks ai serverless inference models. """ +import math from datetime import datetime from typing import Final @@ -15,6 +16,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import apply_off_peak_pricin from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info +NO_CACHE_READ_RATE: Final = float("nan") + # Extract the number of billion parameters from the model name # only used for together_computer LLMs @@ -78,15 +81,15 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ model_info: Final = _resolve_model_info(model) - standard_input_rate: Final[float] = model_info["input_cost_per_token"] or 0.0 standard_cache_read_rate: Final = model_info.get("cache_read_input_token_cost") - input_rate, output_rate, cache_read_rate = apply_off_peak_pricing( + input_rate, output_rate, cache_read_rate_or_unset = apply_off_peak_pricing( model_info, current_time, - standard_input_rate, + model_info["input_cost_per_token"] or 0.0, model_info["output_cost_per_token"] or 0.0, - standard_cache_read_rate if standard_cache_read_rate is not None else standard_input_rate, + standard_cache_read_rate if standard_cache_read_rate is not None else NO_CACHE_READ_RATE, ) + cache_read_rate: Final[float] = input_rate if math.isnan(cache_read_rate_or_unset) else cache_read_rate_or_unset prompt_tokens_details: Final = usage.prompt_tokens_details cached_tokens: Final[int] = ( diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index 21ee56a7873..555fdf7e11d 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -78,14 +78,14 @@ STANDARD_OUTPUT_COST = 6e-07 STANDARD_CACHE_READ_COST = 1.5e-08 -def _register_off_peak_model(off_peak_pricing: dict) -> None: +def _register_off_peak_model(off_peak_pricing: dict, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None: litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = { "litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": STANDARD_INPUT_COST, "output_cost_per_token": STANDARD_OUTPUT_COST, - "cache_read_input_token_cost": STANDARD_CACHE_READ_COST, "off_peak_pricing": off_peak_pricing, + **({} if cache_read_cost is None else {"cache_read_input_token_cost": cache_read_cost}), } @@ -129,6 +129,25 @@ def test_off_peak_rates_left_unset_keep_the_standard_rates(): assert math.isclose(completion_cost, 200 * STANDARD_OUTPUT_COST, rel_tol=1e-10) +def test_off_peak_window_bills_cached_tokens_at_the_off_peak_input_rate_without_a_cache_read_rate(): + """Most fireworks_ai price-map entries carry no cache_read_input_token_cost, so cached tokens + fall back to the input rate, and inside the window that has to be the off-peak one.""" + _register_off_peak_model( + {"hours_utc": OFF_PEAK_WINDOW, "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08}, + cache_read_cost=None, + ) + usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) + + prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=INSIDE_WINDOW) + + assert math.isclose(prompt_cost, 1000 * 1e-08, rel_tol=1e-10) + assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10) + + peak_prompt_cost, _ = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=OUTSIDE_WINDOW) + + assert math.isclose(peak_prompt_cost, 1000 * STANDARD_INPUT_COST, rel_tol=1e-10) + + def test_off_peak_defaults_to_the_current_time(): """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the default current time."""