From 3ad6e5ffc102ddd76f778622c60d86ee8529291d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Mar 2024 10:46:32 -0700 Subject: [PATCH 1/2] (feat) start proxy with default num_workers=1 --- litellm/proxy/proxy_cli.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index e5bcff646b7..52125890011 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -16,13 +16,6 @@ from importlib import resources import shutil telemetry = None -default_num_workers = 1 -try: - default_num_workers = os.cpu_count() or 1 - if default_num_workers is not None and default_num_workers > 0: - default_num_workers -= 1 -except: - pass def append_query_params(url, params): @@ -64,7 +57,7 @@ def is_port_in_use(port): @click.option("--port", default=4000, help="Port to bind the server to.", envvar="PORT") @click.option( "--num_workers", - default=default_num_workers, + default=1, help="Number of gunicorn workers to spin up", envvar="NUM_WORKERS", ) From c068a5fb74498a8d1e20bf500cbc262a78687f58 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Mar 2024 11:47:55 -0700 Subject: [PATCH 2/2] (docs) deploying litellm --- docs/my-website/docs/proxy/deploy.md | 30 ++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 3b668e10298..54f151a5827 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -14,12 +14,19 @@ You can find the Dockerfile to build litellm proxy [here](https://github.com/Ber See the latest available ghcr docker image here: https://github.com/berriai/litellm/pkgs/container/litellm -```shell -docker pull ghcr.io/berriai/litellm:main-latest -``` +Your litellm config.yaml should be called `litellm_config.yaml` in the directory you run this command. +The `-v` command will mount that file + +`AZURE_API_KEY` and `AZURE_API_BASE` are not required to start, just examples on how to pass .env vars ```shell -docker run --env-file=.env -v $(pwd)/path/to/config.yaml:/app/config.yaml -p 4000:4000 ghcr.io/berriai/litellm:main-latest --config config.yaml +docker run \ + -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -e AZURE_API_KEY=d6*********** \ + -e AZURE_API_BASE=https://openai-***********/ \ + -p 4000:4000 \ + ghcr.io/berriai/litellm:main-latest \ + --config /app/config.yaml --detailed_debug ``` @@ -438,6 +445,21 @@ ghcr.io/berriai/litellm-database:main-latest --config your_config.yaml ### 1. Switch of debug logs in production don't use [`--detailed-debug`, `--debug`](https://docs.litellm.ai/docs/proxy/debugging#detailed-debug) or `litellm.set_verbose=True`. We found using debug logs can add 5-10% latency per LLM API call +### 2. Use `run_gunicorn` and `num_workers` + +Example setting `--run_gunicorn` and `--num_workers` +```shell +docker run ghcr.io/berriai/litellm-database:main-latest --run_gunicorn --num_workers 4 +``` + +Why `Gunicorn`? +- Gunicorn takes care of running multiple instances of your web application +- Gunicorn is ideal for running litellm proxy on cluster of machines with Kubernetes + +Why `num_workers`? +Setting `num_workers` to the number of CPUs available ensures optimal utilization of system resources by matching the number of worker processes to the available CPU cores. + + ## Advanced Deployment Settings ### Customization of the server root path