diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a6e26abdf30..908226dc120 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -39,6 +39,8 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( CostCalculatorUtils, calculate_cache_writing_cost, calculate_cost_component, + cost_map_entry_prices_anything, + cost_map_key_prices_anything, generic_cost_per_character, generic_cost_per_token, get_batch_cost_rates, @@ -48,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( get_token_type_cost_breakdown, get_vertex_regional_endpoint_uplift, parse_prompt_tokens_details, + raw_cost_map_entry, select_cost_metric_for_model, ) from litellm.llms.anthropic.cost_calculation import ( @@ -75,6 +78,9 @@ from litellm.llms.deepseek.cost_calculator import ( from litellm.llms.fireworks_ai.cost_calculator import ( cost_per_token as fireworks_ai_cost_per_token, ) +from litellm.llms.fireworks_ai.cost_calculator import ( + has_size_bucket as fireworks_ai_has_size_bucket, +) from litellm.llms.gemini.cost_calculator import cost_per_token as gemini_cost_per_token from litellm.llms.lemonade.cost_calculator import ( cost_per_token as lemonade_cost_per_token, @@ -858,27 +864,8 @@ def _get_hidden_str_for_cost_calc(hidden_params: object, key: str) -> str | None return value if isinstance(value, str) and value else None -_NON_TOKEN_RATE_FIELDS: Final = frozenset( - { - "cost_per_second", - "input_cost_per_second", - "output_cost_per_second", - "input_cost_per_query", - "input_cost_per_text_message", - "tiered_pricing", - } -) - - -def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool: - return any( - value is not None and (field in _NON_TOKEN_RATE_FIELDS or ("cost_per" in field and "token" in field)) - for field, value in entry.items() - ) - - def _bedrock_registration_has_prices(entry: Mapping[str, object]) -> bool: - return _cost_map_entry_prices_anything(entry) or any( + return cost_map_entry_prices_anything(entry) or any( value is not None and field.startswith("cache_") and is_server_derived_pricing_key(field) for field, value in entry.items() ) @@ -908,11 +895,6 @@ def _bedrock_pricing_info( return model_info_with_pricing_overrides(profile, registration[1]) -def cost_map_key_prices_anything(key: str) -> bool: - entry: Final = _raw_cost_map_entry(key) - return entry is not None and _cost_map_entry_prices_anything(entry) - - def _declares_rates(model_info: ModelInfo | None) -> bool: return model_info is not None and cost_map_key_prices_anything(model_info["key"]) @@ -971,6 +953,7 @@ def select_model_name_for_cost_calc( completion_response_model: Final = _get_response_model(completion_response) hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None) provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model") + requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name") explicit_pricing: Final = custom_pricing is True or base_model is not None configured_model: Final = model or ( completion_response_model if isinstance(completion_response_model, str) else None @@ -989,7 +972,7 @@ def select_model_name_for_cost_calc( if custom_pricing is True: if router_model_id is not None and router_model_id in litellm.model_cost: entry: Final = litellm.model_cost[router_model_id] - if _cost_map_entry_prices_anything(entry): + if cost_map_entry_prices_anything(entry): return_model = router_model_id else: return_model = model @@ -1000,14 +983,18 @@ def select_model_name_for_cost_calc( return_model = base_model if base_model is not None else bedrock_pricing_model elif provider_response_model is not None: - return _served_or_priced_requested_model(provider_response_model, model, custom_llm_provider, priced_region) + return _served_or_priced_requested_model( + provider_response_model, requested_model, custom_llm_provider, priced_region + ) elif completion_response_model is None and hidden_params is not None: if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0: return_model = hidden_params.get("model", model) if return_model is None and completion_response_model is not None: - return _served_or_priced_requested_model(completion_response_model, model, custom_llm_provider, priced_region) + return _served_or_priced_requested_model( + completion_response_model, requested_model, custom_llm_provider, priced_region + ) if return_model is None and model is not None: return_model = model @@ -1033,11 +1020,15 @@ def _row_declares_rates(model: str, custom_llm_provider: str | None) -> bool: def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool: rule_info: Final = unmapped_model_generalization(model, custom_llm_provider) - return rule_info is not None and _cost_map_entry_prices_anything(rule_info) + return rule_info is not None and cost_map_entry_prices_anything(rule_info) def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool: - return _row_declares_rates(model, custom_llm_provider) or _rule_declares_rates(model, custom_llm_provider) + return ( + _row_declares_rates(model, custom_llm_provider) + or _rule_declares_rates(model, custom_llm_provider) + or (custom_llm_provider == "fireworks_ai" and fireworks_ai_has_size_bucket(model)) + ) def _served_or_priced_requested_model( @@ -1049,7 +1040,7 @@ def _served_or_priced_requested_model( requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None) if requested == served or _row_declares_rates(served, custom_llm_provider): return served - if not _resolves_to_rates(requested, custom_llm_provider): + if not _resolves_to_rates(requested, _get_provider_for_cost_calc(requested, custom_llm_provider)): return served return served if _rule_declares_rates(served, custom_llm_provider) else requested @@ -2382,7 +2373,7 @@ def _deployment_model_info( ) -> ModelInfo | None: if not custom_pricing: return None - registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None + registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None if registered_deployment_info is not None: return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id if litellm_logging_obj is None: @@ -2444,7 +2435,7 @@ def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool: if model is None: return False rule_info: Final = unmapped_model_generalization(model, pricing_provider) - return rule_info is not None and not _cost_map_entry_prices_anything(rule_info) + return rule_info is not None and not cost_map_entry_prices_anything(rule_info) def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]: @@ -2455,11 +2446,6 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, . ) -def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: - raw_entry: Final = litellm.model_cost.get(key) - return raw_entry if isinstance(raw_entry, Mapping) else None - - def pricing_entry_for_cost_calc( model: str | None, completion_response: object | None, @@ -2499,7 +2485,7 @@ def pricing_entry_for_cost_calc( ) if resolved is None: return None - return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved def ocr_cost( @@ -3491,7 +3477,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) litellm.model_cost.get(model_name), litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"), ) - return any(entry is not None and _cost_map_entry_prices_anything(entry) for entry in entries) + return any(entry is not None and cost_map_entry_prices_anything(entry) for entry in entries) def _first_priced_realtime_token_costs( diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 3b73f65c917..b1cdb831309 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -11,10 +11,10 @@ from pydantic_core import to_jsonable_python from typing_extensions import assert_never import litellm -from litellm.cost_calculator import cost_map_key_prices_anything from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_map_key_prices_anything from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_urls_to_base64, convert_url_to_base64, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3b49f239afe..0413e8217c9 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -52,6 +52,35 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) _DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields) +_NON_TOKEN_RATE_FIELDS: Final = frozenset( + { + "cost_per_second", + "input_cost_per_second", + "output_cost_per_second", + "input_cost_per_query", + "input_cost_per_text_message", + "tiered_pricing", + } +) + + +def cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool: + return any( + value is not None and (field in _NON_TOKEN_RATE_FIELDS or ("cost_per" in field and "token" in field)) + for field, value in entry.items() + ) + + +def raw_cost_map_entry(key: str) -> Mapping[str, object] | None: + raw_entry: Final = litellm.model_cost.get(key) + return raw_entry if isinstance(raw_entry, Mapping) else None + + +def cost_map_key_prices_anything(key: str) -> bool: + entry: Final = raw_cost_map_entry(key) + return entry is not None and cost_map_entry_prices_anything(entry) + + _BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost" _CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType( diff --git a/litellm/llms/fireworks_ai/cost_calculator.py b/litellm/llms/fireworks_ai/cost_calculator.py index d42b4ddbbf7..e36c9616021 100644 --- a/litellm/llms/fireworks_ai/cost_calculator.py +++ b/litellm/llms/fireworks_ai/cost_calculator.py @@ -11,7 +11,7 @@ from litellm.constants import ( FIREWORKS_AI_56_B_MOE, FIREWORKS_AI_176_B_MOE, ) -from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.litellm_core_utils.llm_cost_calc.utils import cost_map_key_prices_anything, generic_cost_per_token from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -56,11 +56,16 @@ def get_base_model_for_pricing(model_name: str) -> str: return "fireworks-ai-default" +def has_size_bucket(model_name: str) -> bool: + return get_base_model_for_pricing(model_name) != "fireworks-ai-default" + + def _resolve_model_info(model: str) -> ModelInfo: try: - return get_model_info(model=model, custom_llm_provider="fireworks_ai") + model_info: Final = get_model_info(model=model, custom_llm_provider="fireworks_ai") except Exception: return _resolve_routed_model_info(model) + return model_info if cost_map_key_prices_anything(model_info["key"]) else _resolve_routed_model_info(model) def _resolve_routed_model_info(model: str) -> ModelInfo: diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e786753e1a8..654d17512bd 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -12104,6 +12104,29 @@ def test_get_model_cost_information_custom_pricing_uses_base_model(): assert result["model_map_key"] != "invoke_test_claude" +@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +def test_get_model_cost_information_maps_requested_model_when_response_names_an_unmapped_display_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + response: Final = ModelResponse(model="Sol Display Max", choices=[]) + response._hidden_params["provider_response_model"] = "Sol Display Max" + response._hidden_params["litellm_model_name"] = requested_model + + result: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=None, custom_pricing=False, custom_llm_provider="fireworks_ai", init_response_obj=response + ) + + assert result["model_map_key"] == requested_model + assert result["model_map_value"] is not None + assert result["model_map_value"]["input_cost_per_token"] == litellm.model_cost[requested_model]["input_cost_per_token"] + + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_deployment_when_no_base_model(): """metadata["deployment"] is used for cost-map lookup when base_model is not set.""" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index be6f178d674..ac7ba6ea06b 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -13,7 +13,7 @@ from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel import litellm -from litellm import TranscriptionResponse, model_cost +from litellm import Router, TranscriptionResponse, model_cost from litellm.constants import REALTIME_METERED_USAGE_KEY from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details @@ -544,6 +544,84 @@ def test_completion_cost_keeps_response_pricing_when_requested_alias_has_no_rate assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[size_bucket], usage)) +@pytest.mark.parametrize("model_passed", [True, False]) +def test_completion_cost_bills_requested_size_bucket_when_response_names_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, model_passed: bool +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-70b-test" + size_bucket: Final = get_base_model_for_pricing(requested_model) + monkeypatch.setitem( + litellm.model_cost, + size_bucket, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="Sol Display Max", choices=[], usage=usage) + response._hidden_params["litellm_model_name"] = requested_model + + cost: Final = ( + completion_cost(completion_response=response, model=requested_model, custom_llm_provider="fireworks_ai") + if model_passed + else completion_cost(completion_response=response) + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[size_bucket], usage)) + + +@pytest.mark.parametrize("served_model", ["Sol Display Max", "accounts/fireworks/models/sol-unmapped-70b-test"]) +def test_completion_cost_bills_size_bucket_for_a_router_deployment_of_an_unmapped_fireworks_model( + _local_model_cost_map: None, served_model: str +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-unmapped-70b-test" + Router(model_list=[{"model_name": "fw-sol", "litellm_params": {"model": requested_model, "api_key": "sk-test"}}]) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model=served_model, choices=[], usage=usage) + response._hidden_params["litellm_model_name"] = requested_model + + cost: Final = completion_cost(completion_response=response) + + size_bucket_row: Final = litellm.model_cost[get_base_model_for_pricing(requested_model)] + assert cost == pytest.approx(_token_rates_cost(size_bucket_row, usage)) + assert cost > 0 + + +def test_completion_cost_bills_requested_model_priced_only_by_a_rule_when_response_row_has_no_rates( + _local_model_cost_map: None, _priced_claude_family_generalization: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "anthropic/sol-served-snapshot-test", + {"litellm_provider": "anthropic", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": None}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-served-snapshot-test", choices=[], usage=usage), + model="anthropic/claude-sol-9", + custom_llm_provider="anthropic", + ) + + assert cost == pytest.approx(usage.prompt_tokens * 0.0000321 + usage.completion_tokens * 0.0000765) + + +def test_completion_cost_bills_requested_model_from_hidden_params_when_no_model_is_passed( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="Sol Display Max", choices=[], usage=usage) + response._hidden_params["litellm_model_name"] = requested_model + + cost: Final = completion_cost(completion_response=response) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + def test_completion_cost_orders_candidates_without_extra_huggingface_config_requests( _local_model_cost_map: None, ) -> None: