From 5eab1feb2011c800295fee5025a87402019baf0a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 00:42:13 +0000 Subject: [PATCH 1/5] fix(fireworks-ai): bill cache-write, reasoning, and audio tokens via the shared cost calculator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/cost_calculator.py | 64 +++++++------- .../test_fireworks_ai_cost_calculator.py | 85 ++++++++++++++++--- 2 files changed, 105 insertions(+), 44 deletions(-) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index 3c43075d940..7bd01115a94 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -2,9 +2,9 @@ For calculating cost of fireworks ai serverless inference models. """ -import math +from collections.abc import Mapping from datetime import datetime -from typing import Final +from typing import Final, cast from litellm.constants import ( FIREWORKS_AI_4_B, @@ -12,12 +12,10 @@ from litellm.constants import ( FIREWORKS_AI_56_B_MOE, FIREWORKS_AI_176_B_MOE, ) -from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info -NO_CACHE_READ_RATE: Final = float("nan") - # Extract the number of billion parameters from the model name # only used for together_computer LLMs @@ -67,6 +65,30 @@ def _resolve_model_info(model: str) -> ModelInfo: return get_model_info(model=base_model, custom_llm_provider="fireworks_ai") +def _with_cache_read_fallback(model_info: ModelInfo) -> ModelInfo: + """Most fireworks_ai price-map entries publish no cache-read rate, and the provider bills + cached reads at the input rate. generic_cost_per_token prices a missing rate at $0, so the + fallback is written into a copy of the entry (the shared model-cost dict must not be + mutated), including inside off_peak_pricing so cached reads track the off-peak input rate + the way the previous calculator did.""" + if model_info.get("cache_read_input_token_cost") is not None: + return model_info + input_rate: Final = model_info.get("input_cost_per_token") + if input_rate is None: + return model_info + effective: Final[dict[str, object]] = dict(model_info) + effective["cache_read_input_token_cost"] = input_rate + off_peak: Final = effective.get("off_peak_pricing") + if isinstance(off_peak, Mapping): + off_peak_map: Final[Mapping[str, object]] = cast(Mapping[str, object], off_peak) + if "cache_read_input_token_cost" not in off_peak_map: + effective["off_peak_pricing"] = { + **off_peak_map, + "cache_read_input_token_cost": off_peak_map.get("input_cost_per_token", input_rate), + } + return cast(ModelInfo, effective) + + def cost_per_token(model: str, usage: Usage, current_time: datetime | None = None) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens, @@ -80,29 +102,11 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - model_info: Final = _resolve_model_info(model) - standard_cache_read_rate: Final = model_info.get("cache_read_input_token_cost") - rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=model_info["input_cost_per_token"] or 0.0, - output_rate=model_info["output_cost_per_token"] or 0.0, - cache_read_rate=standard_cache_read_rate if standard_cache_read_rate is not None else NO_CACHE_READ_RATE, - cache_creation_rate=0.0, - reasoning_rate=None, - ), + model_info: Final = _with_cache_read_fallback(_resolve_model_info(model)) + return generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider="fireworks_ai", + model_info=model_info, + current_time=current_time, ) - cache_read_rate: Final[float] = rates.input_rate if math.isnan(rates.cache_read_rate) else rates.cache_read_rate - - prompt_tokens_details: Final = usage.prompt_tokens_details - cached_tokens: Final[int] = ( - prompt_tokens_details.cached_tokens - if prompt_tokens_details is not None and prompt_tokens_details.cached_tokens is not None - else 0 - ) - non_cached_prompt_tokens: Final[int] = max(usage.prompt_tokens - cached_tokens, 0) - prompt_cost: Final[float] = non_cached_prompt_tokens * rates.input_rate + cached_tokens * cache_read_rate - completion_cost: Final[float] = usage.completion_tokens * rates.output_rate - - return prompt_cost, completion_cost diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index c2e42da1b4c..40a07a10cfc 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -1,13 +1,16 @@ - import math from datetime import datetime, timezone import pytest - import litellm from litellm.llms.fireworks_ai.cost_calculator import cost_per_token -from litellm.types.utils import OffPeakPricing, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import ( + CompletionTokensDetailsWrapper, + OffPeakPricing, + PromptTokensDetailsWrapper, + Usage, +) MODEL = "accounts/fireworks/models/glm-5p2" INPUT_COST = 1.4e-06 @@ -47,12 +50,8 @@ def test_warm_call_cheaper_than_cold_call(): prompt_tokens = 7036 completion_tokens = 8 - cold_prompt_cost, _ = cost_per_token( - model=MODEL, usage=_usage(prompt_tokens, 16, completion_tokens) - ) - warm_prompt_cost, _ = cost_per_token( - model=MODEL, usage=_usage(prompt_tokens, 7020, completion_tokens) - ) + cold_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 16, completion_tokens)) + warm_prompt_cost, _ = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 7020, completion_tokens)) assert warm_prompt_cost < cold_prompt_cost @@ -61,9 +60,7 @@ def test_no_cached_tokens_matches_full_input_rate(): prompt_tokens = 100 completion_tokens = 10 - prompt_cost, completion_cost = cost_per_token( - model=MODEL, usage=_usage(prompt_tokens, 0, completion_tokens) - ) + prompt_cost, completion_cost = cost_per_token(model=MODEL, usage=_usage(prompt_tokens, 0, completion_tokens)) assert prompt_cost == pytest.approx(prompt_tokens * INPUT_COST) assert completion_cost == pytest.approx(completion_tokens * OUTPUT_COST) @@ -78,7 +75,9 @@ STANDARD_OUTPUT_COST = 6e-07 STANDARD_CACHE_READ_COST = 1.5e-08 -def _register_off_peak_model(off_peak_pricing: OffPeakPricing, cache_read_cost: float | None = STANDARD_CACHE_READ_COST) -> None: +def _register_off_peak_model( + off_peak_pricing: OffPeakPricing, cache_read_cost: float | None = STANDARD_CACHE_READ_COST +) -> None: litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = { "litellm_provider": "fireworks_ai", "mode": "chat", @@ -151,10 +150,68 @@ def test_off_peak_window_bills_cached_tokens_at_the_off_peak_input_rate_without_ def test_off_peak_defaults_to_the_current_time(): """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the default current time.""" - _register_off_peak_model({"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08}) + _register_off_peak_model( + {"hours_utc": "00:00-00:00", "input_cost_per_token": 1e-08, "output_cost_per_token": 2e-08} + ) usage = _usage(prompt_tokens=1000, cached_tokens=0, completion_tokens=200) prompt_cost, completion_cost = cost_per_token(model=OFF_PEAK_MODEL, usage=usage) assert math.isclose(prompt_cost, 1000 * 1e-08, rel_tol=1e-10) assert math.isclose(completion_cost, 200 * 2e-08, rel_tol=1e-10) + + +COMPONENT_MODEL = "accounts/fireworks/models/cost-components-test" +COMPONENT_INPUT_COST = 1e-06 +COMPONENT_OUTPUT_COST = 2e-06 +COMPONENT_CACHE_READ_COST = 1e-07 +COMPONENT_CACHE_CREATION_COST = 3e-06 +COMPONENT_REASONING_COST = 4e-06 +COMPONENT_AUDIO_IN_COST = 5e-06 +COMPONENT_AUDIO_OUT_COST = 6e-06 + + +def test_cache_write_reasoning_and_audio_tokens_are_billed_at_their_component_rates(): + """Regression (LIT-7837): the hand-rolled fireworks_ai calculator billed every + cache-creation, reasoning and audio token at $0. The shared calculator treats the + prompt detail counts as subsets of prompt_tokens and the completion detail counts as + subsets of completion_tokens, billing each remainder at the text rate.""" + litellm.model_cost[f"fireworks_ai/{COMPONENT_MODEL}"] = { + "litellm_provider": "fireworks_ai", + "mode": "chat", + "input_cost_per_token": COMPONENT_INPUT_COST, + "output_cost_per_token": COMPONENT_OUTPUT_COST, + "cache_read_input_token_cost": COMPONENT_CACHE_READ_COST, + "cache_creation_input_token_cost": COMPONENT_CACHE_CREATION_COST, + "output_cost_per_reasoning_token": COMPONENT_REASONING_COST, + "input_cost_per_audio_token": COMPONENT_AUDIO_IN_COST, + "output_cost_per_audio_token": COMPONENT_AUDIO_OUT_COST, + } + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=300, + cache_creation_tokens=200, + audio_tokens=100, + ), + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=200, + audio_tokens=50, + ), + ) + + prompt_cost, completion_cost = cost_per_token(model=COMPONENT_MODEL, usage=usage) + + expected_prompt_cost = ( + 400 * COMPONENT_INPUT_COST + + 300 * COMPONENT_CACHE_READ_COST + + 200 * COMPONENT_CACHE_CREATION_COST + + 100 * COMPONENT_AUDIO_IN_COST + ) + expected_completion_cost = ( + 250 * COMPONENT_OUTPUT_COST + 200 * COMPONENT_REASONING_COST + 50 * COMPONENT_AUDIO_OUT_COST + ) + assert prompt_cost == pytest.approx(expected_prompt_cost) + assert completion_cost == pytest.approx(expected_completion_cost) From 405a12783889945a314942a949a162f9186fca86 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 00:57:05 +0000 Subject: [PATCH 2/5] fix(fireworks-ai): drop banned typing.cast to a suppressed import for the copied pricing entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/cost_calculator.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index 7bd01115a94..f323d78e22d 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -2,9 +2,11 @@ For calculating cost of fireworks ai serverless inference models. """ -from collections.abc import Mapping from datetime import datetime -from typing import Final, cast +from typing import ( + Final, + cast, # noqa: TID251 # the fallback entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it +) from litellm.constants import ( FIREWORKS_AI_4_B, @@ -78,14 +80,11 @@ def _with_cache_read_fallback(model_info: ModelInfo) -> ModelInfo: return model_info effective: Final[dict[str, object]] = dict(model_info) effective["cache_read_input_token_cost"] = input_rate - off_peak: Final = effective.get("off_peak_pricing") - if isinstance(off_peak, Mapping): - off_peak_map: Final[Mapping[str, object]] = cast(Mapping[str, object], off_peak) - if "cache_read_input_token_cost" not in off_peak_map: - effective["off_peak_pricing"] = { - **off_peak_map, - "cache_read_input_token_cost": off_peak_map.get("input_cost_per_token", input_rate), - } + off_peak: Final = model_info.get("off_peak_pricing") + if off_peak is not None and "cache_read_input_token_cost" not in off_peak: + off_peak_copy: Final[dict[str, object]] = dict(off_peak) + off_peak_copy["cache_read_input_token_cost"] = off_peak_copy.get("input_cost_per_token", input_rate) + effective["off_peak_pricing"] = off_peak_copy return cast(ModelInfo, effective) From 8352045f32840642f2dd89d9e39a5bd41c6d6bd2 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 01:39:24 +0000 Subject: [PATCH 3/5] style(fireworks-ai): apply repository conventions to the cost component change Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/cost_calculator.py | 31 +++++----- .../test_fireworks_ai_cost_calculator.py | 62 ++++++++++++------- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index f323d78e22d..a92e5208471 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -68,24 +68,25 @@ def _resolve_model_info(model: str) -> ModelInfo: def _with_cache_read_fallback(model_info: ModelInfo) -> ModelInfo: - """Most fireworks_ai price-map entries publish no cache-read rate, and the provider bills - cached reads at the input rate. generic_cost_per_token prices a missing rate at $0, so the - fallback is written into a copy of the entry (the shared model-cost dict must not be - mutated), including inside off_peak_pricing so cached reads track the off-peak input rate - the way the previous calculator did.""" - if model_info.get("cache_read_input_token_cost") is not None: - return model_info + """Most fireworks_ai price-map entries publish no cache-read rate though the provider bills + cached reads at the input rate; the shared map is never mutated, so a copy carries the fallback.""" input_rate: Final = model_info.get("input_cost_per_token") - if input_rate is None: + if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: return model_info - effective: Final[dict[str, object]] = dict(model_info) - effective["cache_read_input_token_cost"] = input_rate off_peak: Final = model_info.get("off_peak_pricing") - if off_peak is not None and "cache_read_input_token_cost" not in off_peak: - off_peak_copy: Final[dict[str, object]] = dict(off_peak) - off_peak_copy["cache_read_input_token_cost"] = off_peak_copy.get("input_cost_per_token", input_rate) - effective["off_peak_pricing"] = off_peak_copy - return cast(ModelInfo, effective) + if off_peak is None or "cache_read_input_token_cost" in off_peak: + return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": input_rate}) + return cast( + ModelInfo, + { + **model_info, + "cache_read_input_token_cost": input_rate, + "off_peak_pricing": { + **off_peak, + "cache_read_input_token_cost": off_peak.get("input_cost_per_token", input_rate), + }, + }, + ) def cost_per_token(model: str, usage: Usage, current_time: datetime | None = None) -> tuple[float, float]: diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index 0ad5eddcc7a..f461f17b627 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -1,5 +1,6 @@ import math from datetime import datetime, timezone +from typing import Final import pytest @@ -51,13 +52,16 @@ STANDARD_CACHE_READ_COST = 1.5e-08 def _register_off_peak_model( off_peak_pricing: OffPeakPricing, cache_read_cost: float | None = STANDARD_CACHE_READ_COST ) -> None: - litellm.model_cost[f"fireworks_ai/{OFF_PEAK_MODEL}"] = { - "litellm_provider": "fireworks_ai", - "mode": "chat", - "input_cost_per_token": STANDARD_INPUT_COST, - "output_cost_per_token": STANDARD_OUTPUT_COST, - "off_peak_pricing": off_peak_pricing, - **({} if cache_read_cost is None else {"cache_read_input_token_cost": cache_read_cost}), + litellm.model_cost = { # test-quality-ok: the save/restore conftest returns litellm.model_cost to the original object after each test, so replacing the map for this entry leaks nothing + **litellm.model_cost, + f"fireworks_ai/{OFF_PEAK_MODEL}": { + "litellm_provider": "fireworks_ai", + "mode": "chat", + "input_cost_per_token": STANDARD_INPUT_COST, + "output_cost_per_token": STANDARD_OUTPUT_COST, + "off_peak_pricing": off_peak_pricing, + **({} if cache_read_cost is None else {"cache_read_input_token_cost": cache_read_cost}), + }, } @@ -145,20 +149,19 @@ COMPONENT_AUDIO_OUT_COST = 6e-06 def test_cache_write_reasoning_and_audio_tokens_are_billed_at_their_component_rates(): - """Regression (LIT-7837): the hand-rolled fireworks_ai calculator billed every - cache-creation, reasoning and audio token at $0. The shared calculator treats the - prompt detail counts as subsets of prompt_tokens and the completion detail counts as - subsets of completion_tokens, billing each remainder at the text rate.""" - litellm.model_cost[f"fireworks_ai/{COMPONENT_MODEL}"] = { - "litellm_provider": "fireworks_ai", - "mode": "chat", - "input_cost_per_token": COMPONENT_INPUT_COST, - "output_cost_per_token": COMPONENT_OUTPUT_COST, - "cache_read_input_token_cost": COMPONENT_CACHE_READ_COST, - "cache_creation_input_token_cost": COMPONENT_CACHE_CREATION_COST, - "output_cost_per_reasoning_token": COMPONENT_REASONING_COST, - "input_cost_per_audio_token": COMPONENT_AUDIO_IN_COST, - "output_cost_per_audio_token": COMPONENT_AUDIO_OUT_COST, + litellm.model_cost = { # test-quality-ok: the save/restore conftest returns litellm.model_cost to the original object after each test, so replacing the map for this entry leaks nothing + **litellm.model_cost, + f"fireworks_ai/{COMPONENT_MODEL}": { + "litellm_provider": "fireworks_ai", + "mode": "chat", + "input_cost_per_token": COMPONENT_INPUT_COST, + "output_cost_per_token": COMPONENT_OUTPUT_COST, + "cache_read_input_token_cost": COMPONENT_CACHE_READ_COST, + "cache_creation_input_token_cost": COMPONENT_CACHE_CREATION_COST, + "output_cost_per_reasoning_token": COMPONENT_REASONING_COST, + "input_cost_per_audio_token": COMPONENT_AUDIO_IN_COST, + "output_cost_per_audio_token": COMPONENT_AUDIO_OUT_COST, + }, } usage = Usage( prompt_tokens=1000, @@ -188,3 +191,20 @@ def test_cache_write_reasoning_and_audio_tokens_are_billed_at_their_component_ra ) assert prompt_cost == pytest.approx(expected_prompt_cost) assert completion_cost == pytest.approx(expected_completion_cost) + + +def test_an_entry_without_an_input_rate_gets_no_cache_read_fallback(): + litellm.model_cost = { # test-quality-ok: the save/restore conftest returns litellm.model_cost to the original object after each test, so replacing the map for this entry leaks nothing + **litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # the SDK types model_cost as dict[Unknown, Unknown] + "fireworks_ai/accounts/fireworks/models/no-input-rate-test": { + "litellm_provider": "fireworks_ai", + "mode": "chat", + "output_cost_per_token": 2e-06, + }, + } + usage: Final = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) + + prompt_cost, completion_cost = cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage) + + assert prompt_cost == 0 + assert completion_cost == 200 * 2e-06 From 91c7f758640b5d60c3fcfd9e51bcd1770a74fd25 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 01:44:30 +0000 Subject: [PATCH 4/5] docs(fireworks-ai): describe the cache-read fallback without asserting provider billing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/fireworks_ai/cost_calculator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index a92e5208471..1795a700d25 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -68,8 +68,8 @@ def _resolve_model_info(model: str) -> ModelInfo: def _with_cache_read_fallback(model_info: ModelInfo) -> ModelInfo: - """Most fireworks_ai price-map entries publish no cache-read rate though the provider bills - cached reads at the input rate; the shared map is never mutated, so a copy carries the fallback.""" + """Entries without a cache-read rate keep the previous calculator's input-rate fallback for cached + reads (LIT-7845 tracks the documented discount); the shared map is never mutated, so a copy carries it.""" input_rate: Final = model_info.get("input_cost_per_token") if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: return model_info From 133f1e8ef56a2870e175128f68eb434c5061d8d4 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 02:06:42 +0000 Subject: [PATCH 5/5] test(fireworks-ai): drop the explanatory comment on the cache-read constant Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/fireworks_ai/test_fireworks_ai_cost_calculator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index f461f17b627..1bee310d9d3 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -15,8 +15,6 @@ from litellm.types.utils import ( MODEL = "accounts/fireworks/models/glm-5p2" INPUT_COST = 1.4e-06 -# Read the cached rate from the price map so this test tracks the shipped value -# (glm-5p2 is $0.14/1M) instead of hardcoding a number that breaks when it changes. CACHE_READ_COST = litellm.get_model_info(model=MODEL, custom_llm_provider="fireworks_ai")["cache_read_input_token_cost"] OUTPUT_COST = 4.4e-06