fix(cost): bill fireworks cached tokens at the off-peak input rate when no cache-read rate exists

litellm/llms/fireworks_ai/cost_calculator.py

@@ -2,6 +2,7 @@
 For calculating cost of fireworks ai serverless inference models.
 """
 
+import math
 from datetime import datetime
 from typing import Final
 
@@ -15,6 +16,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import apply_off_peak_pricin
 from litellm.types.utils import ModelInfo, Usage
 from litellm.utils import get_model_info
 
+NO_CACHE_READ_RATE: Final = float("nan")
+
 
 # Extract the number of billion parameters from the model name
 # only used for together_computer LLMs
@@ -78,15 +81,15 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
     """
     model_info: Final = _resolve_model_info(model)
-    standard_input_rate: Final[float] = model_info["input_cost_per_token"] or 0.0
     standard_cache_read_rate: Final = model_info.get("cache_read_input_token_cost")
-    input_rate, output_rate, cache_read_rate = apply_off_peak_pricing(
+    input_rate, output_rate, cache_read_rate_or_unset = apply_off_peak_pricing(
         model_info,
         current_time,
-        standard_input_rate,
+        model_info["input_cost_per_token"] or 0.0,
         model_info["output_cost_per_token"] or 0.0,
-        standard_cache_read_rate if standard_cache_read_rate is not None else standard_input_rate,
+        standard_cache_read_rate if standard_cache_read_rate is not None else NO_CACHE_READ_RATE,
     )
+    cache_read_rate: Final[float] = input_rate if math.isnan(cache_read_rate_or_unset) else cache_read_rate_or_unset
 
     prompt_tokens_details: Final = usage.prompt_tokens_details
     cached_tokens: Final[int] = (

tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py

@@ -78,14 +78,14 @@ STANDARD_OUTPUT_COST = 6e-07
 STANDARD_CACHE_READ_COST = 1.5e-08
 
 
-def _register_off_peak_model(off_peak_pricing: dict) -> None:
+def _register_off_peak_model(off_peak_pricing: dict, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None:
     litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = {
         "litellm_provider": "fireworks_ai",
         "mode": "chat",
         "input_cost_per_token": STANDARD_INPUT_COST,
         "output_cost_per_token": STANDARD_OUTPUT_COST,
-        "cache_read_input_token_cost": STANDARD_CACHE_READ_COST,
         "off_peak_pricing": off_peak_pricing,
+        **({} if cache_read_cost is None else {"cache_read_input_token_cost": cache_read_cost}),
     }
 
 
@@ -129,6 +129,25 @@ def test_off_peak_rates_left_unset_keep_the_standard_rates():
     assert math.isclose(completion_cost, 200 * STANDARD_OUTPUT_COST, rel_tol=1e-10)
 
 
+def test_off_peak_window_bills_cached_tokens_at_the_off_peak_input_rate_without_a_cache_read_rate():
+    """Most fireworks_ai price-map entries carry no cache_read_input_token_cost, so cached tokens
+    fall back to the input rate, and inside the window that has to be the off-peak one."""
+    _register_off_peak_model(
+        {"hours_utc": OFF_PEAK_WINDOW, "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08},
+        cache_read_cost=None,
+    )
+    usage = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200)
+
+    prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=INSIDE_WINDOW)
+
+    assert math.isclose(prompt_cost, 1000 * 1e-08, rel_tol=1e-10)
+    assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10)
+
+    peak_prompt_cost, _ = cost_per_token(model=OFF_PEAK_MODEL, usage=usage, current_time=OUTSIDE_WINDOW)
+
+    assert math.isclose(peak_prompt_cost, 1000 * STANDARD_INPUT_COST, rel_tol=1e-10)
+
+
 def test_off_peak_defaults_to_the_current_time():
     """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the
     default current time."""