diff --git a/litellm/constants.py b/litellm/constants.py index 7ddd16c880c..0655473301f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -14,6 +14,7 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) ) +DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4)) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" SQS_API_VERSION = "2012-11-05" diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index cc4b1652d08..7ddc5b96bfc 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -12,6 +12,8 @@ import click import httpx from dotenv import load_dotenv +from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY + if TYPE_CHECKING: from fastapi import FastAPI else: @@ -308,8 +310,8 @@ class ProxyInitializationHelpers: @click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT") @click.option( "--num_workers", - default=1, - help="Number of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used.", + default=DEFAULT_NUM_WORKERS_LITELLM_PROXY, + help="Number of uvicorn / gunicorn workers to spin up. By default, 4 uvicorn workers are used.", envvar="NUM_WORKERS", ) @click.option("--api_base", default=None, help="API base URL.") diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index be6da159a37..acff522196e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,4 +1,6 @@ model_list: - - model_name: xai/* + - model_name: db-openai-endpoint litellm_params: - model: xai/* + model: openai/* + api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/ + mock_response: "hi"