From c9cd666b368c0194c6d9d3c04bbb77da7b2e630f Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 01:18:09 +0000 Subject: [PATCH] refactor(cost_calc): move the fireworks cache-read default under litellm/llms Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 33 +++--------- litellm/llms/fireworks_ai/cache_pricing.py | 38 ++++++++++++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 25 ++-------- .../test_fireworks_ai_cache_pricing.py | 50 +++++++++++++++++++ 4 files changed, 98 insertions(+), 48 deletions(-) create mode 100644 litellm/llms/fireworks_ai/cache_pricing.py create mode 100644 tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 7de4534ef2c..cf1b1962df8 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -14,11 +14,11 @@ from typing_extensions import ReadOnly import litellm from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( select_tier_for_input, tier_rate, ) +from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, @@ -74,31 +74,12 @@ def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: def apply_provider_cache_read_default(model_info: ModelInfo, custom_llm_provider: str | None) -> ModelInfo: - """Apply provider-specific defaults for cache-read pricing.""" - if custom_llm_provider != "fireworks_ai": - return model_info - input_rate: Final = model_info.get("input_cost_per_token") - if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: - return model_info - cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - off_peak: Final = model_info.get("off_peak_pricing") - if off_peak is None or "cache_read_input_token_cost" in off_peak: - return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate}) - return cast( - ModelInfo, - { - **model_info, - "cache_read_input_token_cost": cache_read_rate, - "off_peak_pricing": { - **off_peak, - "cache_read_input_token_cost": ( - off_peak["input_cost_per_token"] * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - if "input_cost_per_token" in off_peak - else cache_read_rate - ), - }, - }, - ) + """Dispatch to the provider's cache-read pricing default; providers without one keep their entry as is.""" + match custom_llm_provider: + case "fireworks_ai": + return with_default_cache_read_rate(model_info) + case _: + return model_info def _get_token_detail_value(details: object, key: str) -> int | None: diff --git a/litellm/llms/fireworks_ai/cache_pricing.py b/litellm/llms/fireworks_ai/cache_pricing.py new file mode 100644 index 00000000000..c28a8684c66 --- /dev/null +++ b/litellm/llms/fireworks_ai/cache_pricing.py @@ -0,0 +1,38 @@ +""" +Fireworks AI serverless cache-read pricing defaults. +""" + +from typing import ( + Final, + cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it +) + +from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO +from litellm.types.utils import ModelInfo + + +def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo: + """Entries without a cache-read rate get the documented discount off the input rate; the shared map is + never mutated, so a copy carries it.""" + input_rate: Final = model_info.get("input_cost_per_token") + if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: + return model_info + cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO + off_peak: Final = model_info.get("off_peak_pricing") + if off_peak is None or "cache_read_input_token_cost" in off_peak: + return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate}) + return cast( + ModelInfo, + { + **model_info, + "cache_read_input_token_cost": cache_read_rate, + "off_peak_pricing": { + **off_peak, + "cache_read_input_token_cost": ( + off_peak["input_cost_per_token"] * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO + if "input_cost_per_token" in off_peak + else cache_read_rate + ), + }, + }, + ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 32d7dd1d0c2..e85cbe65b18 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -98,36 +98,17 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re assert completion_cost == pytest.approx(380 * 9.7e-7) -def test_apply_provider_cache_read_default_preserves_identity_and_input_data() -> None: +def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: openai_info: ModelInfo = {"input_cost_per_token": 2e-6} - explicit_fireworks_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "cache_read_input_token_cost": 1e-6, - } - fireworks_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - }, - } - original_fireworks_info: ModelInfo = deepcopy(fireworks_info) + fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} assert apply_provider_cache_read_default(openai_info, "openai") is openai_info - assert apply_provider_cache_read_default(explicit_fireworks_info, "fireworks_ai") is explicit_fireworks_info + assert apply_provider_cache_read_default(openai_info, None) is openai_info processed_fireworks_info = apply_provider_cache_read_default(fireworks_info, "fireworks_ai") - assert fireworks_info == original_fireworks_info assert processed_fireworks_info is not fireworks_info assert processed_fireworks_info["cache_read_input_token_cost"] == pytest.approx(2e-6 * 0.5) - assert processed_fireworks_info["off_peak_pricing"] == { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - "cache_read_input_token_cost": 1e-6 * 0.5, - } def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py new file mode 100644 index 00000000000..2ab273b27c6 --- /dev/null +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py @@ -0,0 +1,50 @@ +from copy import deepcopy + +import pytest + +from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO +from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate +from litellm.types.utils import ModelInfo + + +def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None: + explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6} + no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6} + + assert with_default_cache_read_rate(explicit_info) is explicit_info + assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info + + +def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None: + model_info: ModelInfo = { + "input_cost_per_token": 2e-6, + "off_peak_pricing": { + "hours_utc": "14:00-00:00", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 3e-6, + }, + } + original: ModelInfo = deepcopy(model_info) + + derived = with_default_cache_read_rate(model_info) + + assert model_info == original + assert derived is not model_info + assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO) + assert derived["off_peak_pricing"] == { + "hours_utc": "14:00-00:00", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 3e-6, + "cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO, + } + + +def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None: + model_info: ModelInfo = { + "input_cost_per_token": 2e-6, + "off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}, + } + + derived = with_default_cache_read_rate(model_info) + + assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"]