mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
fix(proxy_server.py): handle decrypting model list from db, when litellm params is base model
This commit is contained in:
Krrish Dholakia
parent
54095a256a
commit
338bfcd21b
3 changed files with 63 additions and 38 deletions
|
|
@ -4801,6 +4801,10 @@
|
|||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.25e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
|
|
@ -4827,6 +4831,10 @@
|
|||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.25e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
|
|
@ -19720,6 +19728,10 @@
|
|||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.25e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
|
|
@ -21086,6 +21098,10 @@
|
|||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.25e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"input_cost_per_token_batches": 1.5e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
|
|
@ -21108,6 +21124,10 @@
|
|||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 2.25e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"input_cost_per_token_batches": 1.5e-06,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 200000,
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ def decrypt_value_helper(
|
|||
value: str,
|
||||
key: str, # this is just for debug purposes, showing the k,v pair that's invalid. not a signing key.
|
||||
exception_type: Literal["debug", "error"] = "error",
|
||||
return_original_value: bool = False,
|
||||
):
|
||||
signing_key = _get_salt_key()
|
||||
|
||||
|
|
@ -55,14 +56,14 @@ def decrypt_value_helper(
|
|||
error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {str(e)}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key"
|
||||
if exception_type == "debug":
|
||||
verbose_proxy_logger.debug(error_message)
|
||||
return None
|
||||
return value if return_original_value else None
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
f"Unable to decrypt value={value} for key: {key}, returning None"
|
||||
)
|
||||
verbose_proxy_logger.exception(error_message)
|
||||
# [Non-Blocking Exception. - this should not block decrypting other values]
|
||||
return None
|
||||
return value if return_original_value else None
|
||||
|
||||
|
||||
def encrypt_value(value: str, signing_key: str):
|
||||
|
|
|
|||
|
|
@ -253,9 +253,7 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
|
|||
from litellm.proxy.management_endpoints.internal_user_endpoints import (
|
||||
router as internal_user_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import (
|
||||
user_update,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import (
|
||||
delete_verification_tokens,
|
||||
duration_in_seconds,
|
||||
|
|
@ -302,9 +300,7 @@ from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMi
|
|||
from litellm.proxy.openai_files_endpoints.files_endpoints import (
|
||||
router as openai_files_router,
|
||||
)
|
||||
from litellm.proxy.openai_files_endpoints.files_endpoints import (
|
||||
set_files_config,
|
||||
)
|
||||
from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
passthrough_endpoint_router,
|
||||
)
|
||||
|
|
@ -467,9 +463,9 @@ except ImportError:
|
|||
server_root_path = os.getenv("SERVER_ROOT_PATH", "")
|
||||
_license_check = LicenseCheck()
|
||||
premium_user: bool = _license_check.is_premium()
|
||||
premium_user_data: Optional[
|
||||
"EnterpriseLicenseData"
|
||||
] = _license_check.airgapped_license_data
|
||||
premium_user_data: Optional["EnterpriseLicenseData"] = (
|
||||
_license_check.airgapped_license_data
|
||||
)
|
||||
global_max_parallel_request_retries_env: Optional[str] = os.getenv(
|
||||
"LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES"
|
||||
)
|
||||
|
|
@ -966,9 +962,9 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter(
|
|||
dual_cache=user_api_key_cache
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter)
|
||||
redis_usage_cache: Optional[
|
||||
RedisCache
|
||||
] = None # redis cache used for tracking spend, tpm/rpm limits
|
||||
redis_usage_cache: Optional[RedisCache] = (
|
||||
None # redis cache used for tracking spend, tpm/rpm limits
|
||||
)
|
||||
user_custom_auth = None
|
||||
user_custom_key_generate = None
|
||||
user_custom_sso = None
|
||||
|
|
@ -1299,9 +1295,9 @@ async def update_cache( # noqa: PLR0915
|
|||
_id = "team_id:{}".format(team_id)
|
||||
try:
|
||||
# Fetch the existing cost for the given user
|
||||
existing_spend_obj: Optional[
|
||||
LiteLLM_TeamTable
|
||||
] = await user_api_key_cache.async_get_cache(key=_id)
|
||||
existing_spend_obj: Optional[LiteLLM_TeamTable] = (
|
||||
await user_api_key_cache.async_get_cache(key=_id)
|
||||
)
|
||||
if existing_spend_obj is None:
|
||||
# do nothing if team not in api key cache
|
||||
return
|
||||
|
|
@ -1878,9 +1874,7 @@ class ProxyConfig:
|
|||
f"{blue_color_code}Set Global BitBucket Config on LiteLLM Proxy{reset_color_code}"
|
||||
)
|
||||
elif key == "global_gitlab_config":
|
||||
from litellm.integrations.gitlab import (
|
||||
set_global_gitlab_config,
|
||||
)
|
||||
from litellm.integrations.gitlab import set_global_gitlab_config
|
||||
|
||||
set_global_gitlab_config(value)
|
||||
verbose_proxy_logger.info(
|
||||
|
|
@ -2541,10 +2535,14 @@ class ProxyConfig:
|
|||
_model_list: list = []
|
||||
for m in new_models:
|
||||
_litellm_params = m.litellm_params
|
||||
if isinstance(_litellm_params, BaseModel):
|
||||
_litellm_params = _litellm_params.model_dump()
|
||||
if isinstance(_litellm_params, dict):
|
||||
# decrypt values
|
||||
for k, v in _litellm_params.items():
|
||||
decrypted_value = decrypt_value_helper(value=v, key=k)
|
||||
decrypted_value = decrypt_value_helper(
|
||||
value=v, key=k, return_original_value=True
|
||||
)
|
||||
_litellm_params[k] = decrypted_value
|
||||
_litellm_params = LiteLLM_Params(**_litellm_params)
|
||||
else:
|
||||
|
|
@ -2628,7 +2626,7 @@ class ProxyConfig:
|
|||
) -> None:
|
||||
"""
|
||||
Helper method to add a single callback to litellm for specified event types.
|
||||
|
||||
|
||||
Args:
|
||||
callback: The callback name to add
|
||||
event_types: List of event types (e.g., ["success"], ["failure"], or ["success", "failure"])
|
||||
|
|
@ -3153,10 +3151,10 @@ class ProxyConfig:
|
|||
)
|
||||
|
||||
try:
|
||||
guardrails_in_db: List[
|
||||
Guardrail
|
||||
] = await GuardrailRegistry.get_all_guardrails_from_db(
|
||||
prisma_client=prisma_client
|
||||
guardrails_in_db: List[Guardrail] = (
|
||||
await GuardrailRegistry.get_all_guardrails_from_db(
|
||||
prisma_client=prisma_client
|
||||
)
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
"guardrails from the DB %s", str(guardrails_in_db)
|
||||
|
|
@ -3386,9 +3384,9 @@ async def initialize( # noqa: PLR0915
|
|||
user_api_base = api_base
|
||||
dynamic_config[user_model]["api_base"] = api_base
|
||||
if api_version:
|
||||
os.environ[
|
||||
"AZURE_API_VERSION"
|
||||
] = api_version # set this for azure - litellm can read this from the env
|
||||
os.environ["AZURE_API_VERSION"] = (
|
||||
api_version # set this for azure - litellm can read this from the env
|
||||
)
|
||||
if max_tokens: # model-specific param
|
||||
dynamic_config[user_model]["max_tokens"] = max_tokens
|
||||
if temperature: # model-specific param
|
||||
|
|
@ -3888,10 +3886,10 @@ class ProxyStartupEvent:
|
|||
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS,
|
||||
LITELLM_KEY_ROTATION_ENABLED,
|
||||
)
|
||||
|
||||
|
||||
key_rotation_enabled: Optional[bool] = str_to_bool(LITELLM_KEY_ROTATION_ENABLED)
|
||||
verbose_proxy_logger.debug(f"key_rotation_enabled: {key_rotation_enabled}")
|
||||
|
||||
|
||||
if key_rotation_enabled is True:
|
||||
try:
|
||||
from litellm.proxy.common_utils.key_rotation_manager import (
|
||||
|
|
@ -3902,19 +3900,25 @@ class ProxyStartupEvent:
|
|||
global prisma_client
|
||||
if prisma_client is not None:
|
||||
key_rotation_manager = KeyRotationManager(prisma_client)
|
||||
verbose_proxy_logger.debug(f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)")
|
||||
verbose_proxy_logger.debug(
|
||||
f"Key rotation background job scheduled every {LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_KEY_ROTATION_ENABLED=true)"
|
||||
)
|
||||
scheduler.add_job(
|
||||
key_rotation_manager.process_rotations,
|
||||
"interval",
|
||||
seconds=LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS,
|
||||
id="key_rotation_job"
|
||||
id="key_rotation_job",
|
||||
)
|
||||
else:
|
||||
verbose_proxy_logger.warning("Key rotation enabled but prisma_client not available")
|
||||
verbose_proxy_logger.warning(
|
||||
"Key rotation enabled but prisma_client not available"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(f"Failed to setup key rotation job: {e}")
|
||||
else:
|
||||
verbose_proxy_logger.debug("Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)")
|
||||
verbose_proxy_logger.debug(
|
||||
"Key rotation disabled (set LITELLM_KEY_ROTATION_ENABLED=true to enable)"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def _setup_prisma_client(
|
||||
|
|
@ -8745,9 +8749,9 @@ async def get_config_list(
|
|||
hasattr(sub_field_info, "description")
|
||||
and sub_field_info.description is not None
|
||||
):
|
||||
nested_fields[
|
||||
idx
|
||||
].field_description = sub_field_info.description
|
||||
nested_fields[idx].field_description = (
|
||||
sub_field_info.description
|
||||
)
|
||||
idx += 1
|
||||
|
||||
_stored_in_db = None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue