From 936640948d79d5b7e3f0cf23a0c459134fcc9158 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 5 Aug 2024 15:07:46 -0700 Subject: [PATCH 1/3] fix: bump default allowed_fails + reduce default db pool limit Fixes issues with running proxy server in production --- litellm/__init__.py | 3 +++ litellm/proxy/proxy_cli.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6dc678b3e59..bf084e20118 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -146,6 +146,9 @@ return_response_headers: bool = ( ) ################## logging: bool = True +enable_caching_on_provider_specific_optional_params: bool = ( + False # feature-flag for caching on optional params - e.g. 'top_k' +) caching: bool = ( False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 1d0eef6a0e8..e1bc363d631 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -19,11 +19,17 @@ litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" if litellm_mode == "DEV": load_dotenv() import shutil +from enum import Enum from importlib import resources telemetry = None +class LiteLLMDatabaseConnectionPool(Enum): + database_connection_pool_limit = 10 + database_connection_pool_timeout = 60 + + def append_query_params(url, params) -> str: from litellm._logging import verbose_proxy_logger @@ -526,10 +532,12 @@ def run_server( ) database_url = general_settings.get("database_url", None) db_connection_pool_limit = general_settings.get( - "database_connection_pool_limit", 100 + "database_connection_pool_limit", + LiteLLMDatabaseConnectionPool.database_connection_pool_limit.value, ) db_connection_timeout = general_settings.get( - "database_connection_timeout", 60 + "database_connection_timeout", + LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value, ) if database_url and database_url.startswith("os.environ/"): original_dir = os.getcwd() From 5f6f1bdad4b29061da4ab79c545999b22a8f7e90 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 5 Aug 2024 16:50:26 -0700 Subject: [PATCH 2/3] fix(__init__.py): bump default allowed fails --- litellm/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index bf084e20118..dfc3f3fc1bb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -271,7 +271,7 @@ default_fallbacks: Optional[List] = None fallbacks: Optional[List] = None context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None -allowed_fails: int = 0 +allowed_fails: int = 3 num_retries_per_request: Optional[int] = ( None # for the request overall (incl. fallbacks + model retries) ) From 56aa7c0f4029ce885829b6c4d4636e3894ea7f18 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 5 Aug 2024 20:00:12 -0700 Subject: [PATCH 3/3] build(config.yml): pin prisma version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77942273be4..5712c71ca6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,7 +48,7 @@ jobs: pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 pip install openai==1.34.0 - pip install prisma + pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" pip install fastapi