From 338bfcd21b1582b106061bfcdb768bd53be95798 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 2 Oct 2025 18:07:20 -0700 Subject: [PATCH] fix(proxy_server.py): handle decrypting model list from db, when litellm params is base model --- ...odel_prices_and_context_window_backup.json | 20 +++++ .../common_utils/encrypt_decrypt_utils.py | 5 +- litellm/proxy/proxy_server.py | 76 ++++++++++--------- 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bae7c11e66c..1b8f7f7c08f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4801,6 +4801,10 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -4827,6 +4831,10 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -19720,6 +19728,10 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, @@ -21086,6 +21098,10 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, @@ -21108,6 +21124,10 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 724a46e609b..98207393c7f 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -39,6 +39,7 @@ def decrypt_value_helper( value: str, key: str, # this is just for debug purposes, showing the k,v pair that's invalid. not a signing key. exception_type: Literal["debug", "error"] = "error", + return_original_value: bool = False, ): signing_key = _get_salt_key() @@ -55,14 +56,14 @@ def decrypt_value_helper( error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {str(e)}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" if exception_type == "debug": verbose_proxy_logger.debug(error_message) - return None + return value if return_original_value else None verbose_proxy_logger.debug( f"Unable to decrypt value={value} for key: {key}, returning None" ) verbose_proxy_logger.exception(error_message) # [Non-Blocking Exception. - this should not block decrypting other values] - return None + return value if return_original_value else None def encrypt_value(value: str, signing_key: str): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 55e4a96b5b0..5409d73164b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -253,9 +253,7 @@ from litellm.proxy.management_endpoints.customer_endpoints import ( from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) -from litellm.proxy.management_endpoints.internal_user_endpoints import ( - user_update, -) +from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( delete_verification_tokens, duration_in_seconds, @@ -302,9 +300,7 @@ from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMi from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) -from litellm.proxy.openai_files_endpoints.files_endpoints import ( - set_files_config, -) +from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( passthrough_endpoint_router, ) @@ -467,9 +463,9 @@ except ImportError: server_root_path = os.getenv("SERVER_ROOT_PATH", "") _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional[ - "EnterpriseLicenseData" -] = _license_check.airgapped_license_data +premium_user_data: Optional["EnterpriseLicenseData"] = ( + _license_check.airgapped_license_data +) global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -966,9 +962,9 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter( dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[ - RedisCache -] = None # redis cache used for tracking spend, tpm/rpm limits +redis_usage_cache: Optional[RedisCache] = ( + None # redis cache used for tracking spend, tpm/rpm limits +) user_custom_auth = None user_custom_key_generate = None user_custom_sso = None @@ -1299,9 +1295,9 @@ async def update_cache( # noqa: PLR0915 _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[ - LiteLLM_TeamTable - ] = await user_api_key_cache.async_get_cache(key=_id) + existing_spend_obj: Optional[LiteLLM_TeamTable] = ( + await user_api_key_cache.async_get_cache(key=_id) + ) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -1878,9 +1874,7 @@ class ProxyConfig: f"{blue_color_code}Set Global BitBucket Config on LiteLLM Proxy{reset_color_code}" ) elif key == "global_gitlab_config": - from litellm.integrations.gitlab import ( - set_global_gitlab_config, - ) + from litellm.integrations.gitlab import set_global_gitlab_config set_global_gitlab_config(value) verbose_proxy_logger.info( @@ -2541,10 +2535,14 @@ class ProxyConfig: _model_list: list = [] for m in new_models: _litellm_params = m.litellm_params + if isinstance(_litellm_params, BaseModel): + _litellm_params = _litellm_params.model_dump() if isinstance(_litellm_params, dict): # decrypt values for k, v in _litellm_params.items(): - decrypted_value = decrypt_value_helper(value=v, key=k) + decrypted_value = decrypt_value_helper( + value=v, key=k, return_original_value=True + ) _litellm_params[k] = decrypted_value _litellm_params = LiteLLM_Params(**_litellm_params) else: @@ -2628,7 +2626,7 @@ class ProxyConfig: ) -> None: """ Helper method to add a single callback to litellm for specified event types. - + Args: callback: The callback name to add event_types: List of event types (e.g., ["success"], ["failure"], or ["success", "failure"]) @@ -3153,10 +3151,10 @@ class ProxyConfig: ) try: - guardrails_in_db: List[ - Guardrail - ] = await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client + guardrails_in_db: List[Guardrail] = ( + await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client + ) ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -3386,9 +3384,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ[ - "AZURE_API_VERSION" - ] = api_version # set this for azure - litellm can read this from the env + os.environ["AZURE_API_VERSION"] = ( + api_version # set this for azure - litellm can read this from the env + ) if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -3888,10 +3886,10 @@ class ProxyStartupEvent: LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS, LITELLM_KEY_ROTATION_ENABLED, ) - + key_rotation_enabled: Optional[bool] = str_to_bool(LITELLM_KEY_ROTATION_ENABLED) verbose_proxy_logger.debug(f"key_rotation_enabled: {key_rotation_enabled}") - + if key_rotation_enabled is True: try: from litellm.proxy.common_utils.key_rotation_manager import ( @@ -3902,19 +3900,25 @@ class ProxyStartupEvent: global prisma_client if prisma_client is not None: key_rotation_manager = KeyRotationManager(prisma_client) - verbose_proxy_logger.debug(f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)") + verbose_proxy_logger.debug( + f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)" + ) scheduler.add_job( key_rotation_manager.process_rotations, "interval", seconds=LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS, - id="key_rotation_job" + id="key_rotation_job", ) else: - verbose_proxy_logger.warning("Key rotation enabled but prisma_client not available") + verbose_proxy_logger.warning( + "Key rotation enabled but prisma_client not available" + ) except Exception as e: verbose_proxy_logger.warning(f"Failed to setup key rotation job: {e}") else: - verbose_proxy_logger.debug("Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)") + verbose_proxy_logger.debug( + "Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)" + ) @classmethod async def _setup_prisma_client( @@ -8745,9 +8749,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[ - idx - ].field_description = sub_field_info.description + nested_fields[idx].field_description = ( + sub_field_info.description + ) idx += 1 _stored_in_db = None