From 30ae52c21ed3f2e02b21d4e86a3a2527d9c2d262 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 23 Mar 2024 17:49:00 -0700 Subject: [PATCH 1/4] (docs) using litellm on EKS --- docs/my-website/docs/proxy/deploy.md | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index d25035760c7..10cd2181989 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -233,6 +233,14 @@ Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. | [LiteLLM Database container + PostgresDB + Redis](#litellm-database-container--postgresdb--redis) | + use Virtual Keys + Track Spend + load balance across multiple litellm containers | + +## Machine Specifications to Deploy LiteLLM + +| Service | Spec | CPUs | Memory | Performance | Architecture | Version| +| --- | --- | --- | --- | --- | --- | --- | +| Server | `t2.small`. | `1vCPUs` | `8GB` | avg latency=`57ms`, median latency=`50ms`, Requests per second=`33` | | | +| Redis Cache | - | - | - | - | | 7.0+ Redis Engine| + ## Deploy with Database ### Docker, Kubernetes, Helm Chart @@ -525,6 +533,57 @@ Provide an ssl certificate when starting litellm proxy server ## Platform-specific Guide + + +### Kubernetes - Deploy on EKS + +Step1. Create an EKS Cluster with the following spec + +```shell +eksctl create cluster --name=litellm-cluster --region=us-west-2 --node-type=t2.small +``` + +Step 2. Mount litellm proxy config on kub cluster + +This will mount your local file called `proxy_config.yaml` on kubernetes cluster + +```shell +kubectl create configmap litellm-config --from-file=proxy_config.yaml +``` + +Step 3. Apply `kub.yaml` and `service.yaml` +Clone the following `kub.yaml` and `service.yaml` files and apply locally + +- Use this `kub.yaml` file - [litellm kub.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/kubernetes/kub.yaml) + +- Use this `service.yaml` file - [litellm service.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/kubernetes/service.yaml) + +Apply `kub.yaml` +``` +kubectl apply -f kub.yaml +``` + +Apply `service.yaml` - creates an AWS load balancer to expose the proxy +``` +kubectl apply -f service.yaml + +# service/litellm-service created +``` + +Step 4. Get Proxy Base URL + +```shell +kubectl get services + +# litellm-service LoadBalancer 10.100.6.31 a472dc7c273fd47fd******.us-west-2.elb.amazonaws.com 4000:30374/TCP 63m +``` + +Proxy Base URL = `a472dc7c273fd47fd******.us-west-2.elb.amazonaws.com:4000` + +That's it, now you can start using LiteLLM Proxy + + + From b0c75d405c4bf9d9bbb1908a625da793b3a0b78b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 23 Mar 2024 17:58:39 -0700 Subject: [PATCH 2/4] (feat) update kub.yaml --- deploy/kubernetes/kub.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml index 1ddc0e95f18..b3a9ddd331b 100644 --- a/deploy/kubernetes/kub.yaml +++ b/deploy/kubernetes/kub.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: litellm-deployment spec: - replicas: 5 + replicas: 3 selector: matchLabels: app: litellm @@ -17,17 +17,17 @@ spec: image: ghcr.io/berriai/litellm:main-latest env: - name: AZURE_API_KEY - value: "d699s" + value: "d6f****" - name: AZURE_API_BASE - value: "https://openai/" + value: "https://openai - name: LITELLM_MASTER_KEY value: "sk-1234" - ports: - - containerPort: 4000 + - name: DATABASE_URL + value: "postgresql://ishaan:********* args: - "--config" - "/app/proxy_config.yaml" # Update the path to mount the config file - volumeMounts: # Define volume mount for proxy_config.yaml + volumeMounts: # Define volume mount for proxy_config.yaml - name: config-volume mountPath: /app readOnly: true From 0c17fe9a326b20a044535f3b728be3771eda554e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 23 Mar 2024 17:59:16 -0700 Subject: [PATCH 3/4] (feat) update config.yaml --- litellm/proxy/proxy_config.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 72ccd9b7d9a..dc50067cb5f 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -12,9 +12,6 @@ model_list: api_base: https://exampleopenaiendpoint-production.up.railway.app/ general_settings: master_key: sk-1234 -router_settings: - set_verbose: True - debug_level: "DEBUG" -litellm_settings: - success_callback: ["prometheus"] + disable_spend_logs: true + disable_reset_budget: true num_retries: 2 \ No newline at end of file From 925e785a6ab3806216e2396910e8981ab4ade3ea Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 23 Mar 2024 17:59:54 -0700 Subject: [PATCH 4/4] (fix) update locustfile --- litellm/proxy/proxy_load_test/locustfile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_load_test/locustfile.py b/litellm/proxy/proxy_load_test/locustfile.py index 263c871efda..ebfca80ad30 100644 --- a/litellm/proxy/proxy_load_test/locustfile.py +++ b/litellm/proxy/proxy_load_test/locustfile.py @@ -18,7 +18,10 @@ class MyUser(HttpUser): payload = { "model": "fake-openai-endpoint", "messages": [ - {"role": "system", "content": "You are a chat bot."}, + { + "role": "system", + "content": "this is a very sweet test message from ishaan", + }, {"role": "user", "content": "Hello, how are you?"}, ], # Add more data as necessary