[Perf] LiteLLM Proxy: +400 RPS when using correct amount of CPU cores (#14153)

* bump: DEFAULT_NUM_WORKERS_LITELLM_PROXY

* fix: DEFAULT_NUM_WORKERS_LITELLM_PROXY

litellm/constants.py

@@ -14,6 +14,7 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
     os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 )
+DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4))
 DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
 SQS_SEND_MESSAGE_ACTION = "SendMessage"
 SQS_API_VERSION = "2012-11-05"

litellm/proxy/proxy_cli.py

@@ -12,6 +12,8 @@ import click
 import httpx
 from dotenv import load_dotenv
 
+from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
+
 if TYPE_CHECKING:
     from fastapi import FastAPI
 else:
@@ -308,8 +310,8 @@ class ProxyInitializationHelpers:
 @click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT")
 @click.option(
     "--num_workers",
-    default=1,
-    help="Number of uvicorn / gunicorn workers to spin up. By default, 1 uvicorn is used.",
+    default=DEFAULT_NUM_WORKERS_LITELLM_PROXY,
+    help="Number of uvicorn / gunicorn workers to spin up. By default, 4 uvicorn workers are used.",
     envvar="NUM_WORKERS",
 )
 @click.option("--api_base", default=None, help="API base URL.")

litellm/proxy/proxy_config.yaml

@@ -1,4 +1,6 @@
 model_list:
-  - model_name: xai/*
+  - model_name: db-openai-endpoint
     litellm_params:
-      model: xai/*
+      model: openai/*
+      api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/
+      mock_response: "hi"