diff --git a/deploy/kubernetes/kub.yaml b/deploy/kubernetes/kub.yaml index 1ddc0e95f18..b3a9ddd331b 100644 --- a/deploy/kubernetes/kub.yaml +++ b/deploy/kubernetes/kub.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: litellm-deployment spec: - replicas: 5 + replicas: 3 selector: matchLabels: app: litellm @@ -17,17 +17,17 @@ spec: image: ghcr.io/berriai/litellm:main-latest env: - name: AZURE_API_KEY - value: "d699s" + value: "d6f****" - name: AZURE_API_BASE - value: "https://openai/" + value: "https://openai - name: LITELLM_MASTER_KEY value: "sk-1234" - ports: - - containerPort: 4000 + - name: DATABASE_URL + value: "postgresql://ishaan:********* args: - "--config" - "/app/proxy_config.yaml" # Update the path to mount the config file - volumeMounts: # Define volume mount for proxy_config.yaml + volumeMounts: # Define volume mount for proxy_config.yaml - name: config-volume mountPath: /app readOnly: true diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index acd61e625bd..10924888bed 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -236,6 +236,14 @@ Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. | [LiteLLM Database container + PostgresDB + Redis](#litellm-database-container--postgresdb--redis) | + use Virtual Keys + Track Spend + load balance across multiple litellm containers | + +## Machine Specifications to Deploy LiteLLM + +| Service | Spec | CPUs | Memory | Performance | Architecture | Version| +| --- | --- | --- | --- | --- | --- | --- | +| Server | `t2.small`. | `1vCPUs` | `8GB` | avg latency=`57ms`, median latency=`50ms`, Requests per second=`33` | | | +| Redis Cache | - | - | - | - | | 7.0+ Redis Engine| + ## Deploy with Database ### Docker, Kubernetes, Helm Chart @@ -514,6 +522,57 @@ Provide an ssl certificate when starting litellm proxy server ## Platform-specific Guide + + +### Kubernetes - Deploy on EKS + +Step1. Create an EKS Cluster with the following spec + +```shell +eksctl create cluster --name=litellm-cluster --region=us-west-2 --node-type=t2.small +``` + +Step 2. Mount litellm proxy config on kub cluster + +This will mount your local file called `proxy_config.yaml` on kubernetes cluster + +```shell +kubectl create configmap litellm-config --from-file=proxy_config.yaml +``` + +Step 3. Apply `kub.yaml` and `service.yaml` +Clone the following `kub.yaml` and `service.yaml` files and apply locally + +- Use this `kub.yaml` file - [litellm kub.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/kubernetes/kub.yaml) + +- Use this `service.yaml` file - [litellm service.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/kubernetes/service.yaml) + +Apply `kub.yaml` +``` +kubectl apply -f kub.yaml +``` + +Apply `service.yaml` - creates an AWS load balancer to expose the proxy +``` +kubectl apply -f service.yaml + +# service/litellm-service created +``` + +Step 4. Get Proxy Base URL + +```shell +kubectl get services + +# litellm-service LoadBalancer 10.100.6.31 a472dc7c273fd47fd******.us-west-2.elb.amazonaws.com 4000:30374/TCP 63m +``` + +Proxy Base URL = `a472dc7c273fd47fd******.us-west-2.elb.amazonaws.com:4000` + +That's it, now you can start using LiteLLM Proxy + + + diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 72ccd9b7d9a..dc50067cb5f 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -12,9 +12,6 @@ model_list: api_base: https://exampleopenaiendpoint-production.up.railway.app/ general_settings: master_key: sk-1234 -router_settings: - set_verbose: True - debug_level: "DEBUG" -litellm_settings: - success_callback: ["prometheus"] + disable_spend_logs: true + disable_reset_budget: true num_retries: 2 \ No newline at end of file diff --git a/litellm/proxy/proxy_load_test/locustfile.py b/litellm/proxy/proxy_load_test/locustfile.py index 263c871efda..ebfca80ad30 100644 --- a/litellm/proxy/proxy_load_test/locustfile.py +++ b/litellm/proxy/proxy_load_test/locustfile.py @@ -18,7 +18,10 @@ class MyUser(HttpUser): payload = { "model": "fake-openai-endpoint", "messages": [ - {"role": "system", "content": "You are a chat bot."}, + { + "role": "system", + "content": "this is a very sweet test message from ishaan", + }, {"role": "user", "content": "Hello, how are you?"}, ], # Add more data as necessary