diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e25aa750623..0e5218095fa 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -2,11 +2,11 @@ ## Helper utilities for cost_per_token() import re -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime, timezone, tzinfo from types import MappingProxyType -from typing import Final, Literal, Protocol, TypedDict, cast +from typing import Final, Literal, Protocol, SupportsFloat, TypedDict, cast from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from typing_extensions import ReadOnly @@ -48,12 +48,7 @@ _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset( } ) -_REGIONAL_UPLIFT_MULTIPLIER_GETTERS: Final[Mapping[str, Callable[[ModelInfo], float | None]]] = MappingProxyType( - { - DataResidency.EU.value: lambda model_info: model_info.get("regional_processing_uplift_multiplier_eu"), - DataResidency.US.value: lambda model_info: model_info.get("regional_processing_uplift_multiplier_us"), - } -) +_VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) _DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields) @@ -1427,12 +1422,17 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | if data_residency is None: return 1.0 residency: Final = data_residency.lower() - multiplier_getter: Final = _REGIONAL_UPLIFT_MULTIPLIER_GETTERS.get(residency) - if multiplier_getter is None: + if residency not in _VALID_DATA_RESIDENCIES: return 1.0 - multiplier: Final = multiplier_getter(model_info) + multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}") if multiplier is None: return 1.0 + if not isinstance(multiplier, (str, SupportsFloat)): + verbose_logger.error( + "Invalid regional_processing_uplift_multiplier_%s for model; defaulting to 1.0", + residency, + ) + return 1.0 try: return float(multiplier) except (TypeError, ValueError): diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index e9f36b368b7..a9c665a31d4 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -210,6 +210,7 @@ class KubernetesPodsResponse(BaseModel): CLEARABLE_LITELLM_PARAMS: Final = frozenset({"cache_control_injection_points", "litellm_credential_name"}) NULL_CLEARABLE_LITELLM_PARAMS: Final = frozenset((*SPECIAL_MODEL_INFO_PARAMS, *CLEARABLE_LITELLM_PARAMS)) +_STR_OBJECT_DICT: Final = TypeAdapter(dict[str, object]) async def update_team(*args, **kwargs): @@ -246,12 +247,12 @@ def _model_write_response( ) -> _ProxyModelRow | Mapping[str, object]: if member_write is None: return row - payload: Final = TypeAdapter(dict[str, object]).validate_json(row.model_dump_json()) + payload: Final = _STR_OBJECT_DICT.validate_json(row.model_dump_json()) stored_params: Final = payload.get("litellm_params") params: Final = ( - TypeAdapter(dict[str, object]).validate_json(stored_params) + _STR_OBJECT_DICT.validate_json(stored_params) if isinstance(stored_params, str) - else TypeAdapter(dict[str, object]).validate_python(stored_params) + else _STR_OBJECT_DICT.validate_python(stored_params) ) redacted: Final = redact_credentials_in_payload(params) return { @@ -1347,7 +1348,7 @@ def legacy_update_model_info_changes( value that differs from the stored override resets that override (``None``). """ stored: Final = db_model.model_info.model_dump(exclude_none=True) - requested: Final = TypeAdapter(dict[str, object]).validate_python( + requested: Final = _STR_OBJECT_DICT.validate_python( incoming_model_info.model_dump(exclude_unset=True, exclude_none=True) ) sent: Final = {k: v for k, v in requested.items() if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""} @@ -1399,20 +1400,16 @@ async def _locked_legacy_update_model_info_json( return None stored_model_info: Final[object] = payload.get("model_info") stored: Final = ( - TypeAdapter(dict[str, object]).validate_json(stored_model_info) + _STR_OBJECT_DICT.validate_json(stored_model_info) if isinstance(stored_model_info, str) - else TypeAdapter(dict[str, object]).validate_python(stored_model_info or {}) - ) - return ( - TypeAdapter(dict[str, object]) - .dump_json( - { - **{k: v for k, v in stored.items() if k not in writes}, - **{k: v for k, v in writes.items() if v is not None}, - } - ) - .decode() + else _STR_OBJECT_DICT.validate_python(stored_model_info or {}) ) + return _STR_OBJECT_DICT.dump_json( + { + **{k: v for k, v in stored.items() if k not in writes}, + **{k: v for k, v in writes.items() if v is not None}, + } + ).decode() def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]: @@ -1521,7 +1518,7 @@ def update_db_model( prisma_compatible_model_dict: Final = PrismaCompatibleUpdateDBModel( model_name=merged_model_name, litellm_params=json.dumps(merged_litellm_params), - model_info=TypeAdapter(dict[str, object]).dump_json(merged_model_info).decode(), + model_info=_STR_OBJECT_DICT.dump_json(merged_model_info).decode(), ) if updated_patch.blocked is not None: @@ -2057,7 +2054,7 @@ async def add_model_to_db( slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None, ) -> "_ProxyModelRow | LiteLLM_ProxyModelTable": # encrypt litellm params # - _litellm_params_dict: Final = TypeAdapter(dict[str, object]).validate_python( + _litellm_params_dict: Final = _STR_OBJECT_DICT.validate_python( model_params.litellm_params.model_dump(exclude_none=True) ) if "complexity_router_config" in _litellm_params_dict: diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 609b28270d7..b4c8c484b87 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_cache_writing_cost, generic_cost_per_token, get_billed_token_rates, + get_regional_uplift_multiplier, get_token_type_cost_breakdown, ) from litellm.llms.gemini.image_generation.cost_calculator import ( @@ -2781,6 +2782,27 @@ def test_fedramp_uplift_defaults_to_one(model_info: ModelInfo, data_residency: s assert get_fedramp_uplift_multiplier(model_info, data_residency) == 1.0 +@pytest.mark.parametrize( + "model_info, data_residency", + [({f"regional_processing_uplift_multiplier_{r.value}": "1.25"}, r.value.upper()) for r in DataResidency], +) +def test_regional_uplift_reads_the_multiplier_named_by_each_residency(model_info: ModelInfo, data_residency: str): + assert get_regional_uplift_multiplier(model_info, data_residency) == 1.25 + + +@pytest.mark.parametrize( + "model_info, data_residency", + [ + ({"regional_processing_uplift_multiplier_eu": "not-a-number"}, "eu"), + ({"regional_processing_uplift_multiplier_eu": [1.25]}, "eu"), + ({"regional_processing_uplift_multiplier_apac": 1.25}, "apac"), + ({"regional_processing_uplift_multiplier_eu": 1.25}, None), + ], +) +def test_regional_uplift_defaults_to_one(model_info: ModelInfo, data_residency: str | None): + assert get_regional_uplift_multiplier(model_info, data_residency) == 1.0 + + def test_service_tier_suffixes_constant_in_sync_with_enum(): from litellm.litellm_core_utils.llm_cost_calc.utils import _SERVICE_TIER_SUFFIXES from litellm.types.utils import ServiceTier