diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 3040fb45d86..d8c39e8b075 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -75,6 +75,8 @@ spec: value: {{ .Release.Name }}-postgresql - name: DATABASE_NAME value: litellm + - name: DATABASE_URL + value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} {{- else if .Values.db.useExisting }} - name: DATABASE_USERNAME valueFrom: diff --git a/deploy/kubernetes/performance-eks/eksctl-cluster.yaml b/deploy/kubernetes/performance-eks/eksctl-cluster.yaml deleted file mode 100644 index 9c74ee39a05..00000000000 --- a/deploy/kubernetes/performance-eks/eksctl-cluster.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# EKS cluster — single worker node. Safe to commit (no secrets). -# -# Create: eksctl create cluster -f deploy/kubernetes/performance-eks/eksctl-cluster.yaml -# Delete: eksctl delete cluster -f deploy/kubernetes/performance-eks/eksctl-cluster.yaml -# -# To allow more nodes later: raise maxSize (and desiredCapacity), then scale or upgrade. - -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig - -metadata: - name: litellm-perf - region: us-east-1 - -managedNodeGroups: - - name: workers - instanceType: t3.xlarge - desiredCapacity: 1 - minSize: 1 - maxSize: 1 diff --git a/deploy/kubernetes/performance-eks/kustomization.yaml b/deploy/kubernetes/performance-eks/kustomization.yaml deleted file mode 100644 index 590e33f9d7f..00000000000 --- a/deploy/kubernetes/performance-eks/kustomization.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Apply from repo root: -# kubectl kustomize deploy/kubernetes/performance-eks --load-restrictor=LoadRestrictionsNone | kubectl apply -f - -# -# Requires Secret litellm-dotenv with DATABASE_URL (Neon) and PROXY_MASTER_KEY — see README. -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: performance-eks - -resources: - - namespace.yaml - - litellm-static-env.yaml - - mock-llm.yaml - - mock-files-api.yaml - - prometheus.yaml - - litellm.yaml - -configMapGenerator: - - name: dev-config - files: - - config.yaml=dev-config.yaml - - name: prometheus-config - files: - - prometheus.yml=../../../prometheus.yml - -generatorOptions: - disableNameSuffixHash: true diff --git a/deploy/kubernetes/performance-eks/litellm-static-env.yaml b/deploy/kubernetes/performance-eks/litellm-static-env.yaml deleted file mode 100644 index 15360ba2efb..00000000000 --- a/deploy/kubernetes/performance-eks/litellm-static-env.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Non-secret env. DATABASE_URL + PROXY_MASTER_KEY: set via Secret litellm-dotenv (required). -apiVersion: v1 -kind: ConfigMap -metadata: - name: litellm-static-env -data: - STORE_MODEL_IN_DB: "True" - ENV: "production" - LITELLM_ENVIRONMENT: "production" \ No newline at end of file diff --git a/deploy/kubernetes/performance-eks/mock-files-api.yaml b/deploy/kubernetes/performance-eks/mock-files-api.yaml deleted file mode 100644 index 81f7ce8daea..00000000000 --- a/deploy/kubernetes/performance-eks/mock-files-api.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# Minimal OpenAI-shaped mock + /health (compose used repo mock_files_api.py on a volume). -apiVersion: v1 -kind: ConfigMap -metadata: - name: mock-files-api-code -data: - mock_files_api.py: | - """Minimal files/OpenAI mock for k8s parity; extend to match your local mock_files_api.py.""" - from fastapi import FastAPI - - app = FastAPI() - - @app.get("/health") - def health(): - return {"status": "ok"} - - @app.get("/v1/models") - def models(): - return {"object": "list", "data": []} ---- -apiVersion: v1 -kind: Service -metadata: - name: mock-files-api -spec: - ports: - - port: 8001 - targetPort: 8001 - selector: - app: mock-files-api ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mock-files-api -spec: - replicas: 1 - selector: - matchLabels: - app: mock-files-api - template: - metadata: - labels: - app: mock-files-api - spec: - containers: - - name: api - image: python:3.11-slim - workingDir: /app - ports: - - containerPort: 8001 - command: - - /bin/sh - - -c - - pip install --no-cache-dir fastapi uvicorn && uvicorn mock_files_api:app --host 0.0.0.0 --port 8001 - volumeMounts: - - name: code - mountPath: /app/mock_files_api.py - subPath: mock_files_api.py - readinessProbe: - httpGet: - path: /health - port: 8001 - initialDelaySeconds: 30 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 12 - volumes: - - name: code - configMap: - name: mock-files-api-code diff --git a/deploy/kubernetes/performance-eks/mock-llm.yaml b/deploy/kubernetes/performance-eks/mock-llm.yaml deleted file mode 100644 index 580302a3ddf..00000000000 --- a/deploy/kubernetes/performance-eks/mock-llm.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mock-llm-provider -spec: - ports: - - port: 80 - targetPort: 80 - selector: - app: mock-llm-provider ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mock-llm-provider -spec: - replicas: 1 - selector: - matchLabels: - app: mock-llm-provider - template: - metadata: - labels: - app: mock-llm-provider - spec: - containers: - - name: httpbun - image: sharat87/httpbun:latest - ports: - - containerPort: 80 - readinessProbe: - httpGet: - path: / - port: 80 - initialDelaySeconds: 3 - periodSeconds: 5 diff --git a/deploy/performance/README.md b/deploy/performance/README.md index 0165e7386d1..57f9bdf247e 100644 --- a/deploy/performance/README.md +++ b/deploy/performance/README.md @@ -1,44 +1,182 @@ -# Performance-related deploy assets +# Performance stack (EKS) -## Full Kustomize stack on EKS (Neon + in-memory cache) +**Primary path: Helm + Argo CD** — `deploy/charts/litellm-helm` with values in `helm/litellm-values.yaml` (bundled **Bitnami Postgres + Redis**, shared **Redis** cache, LoadBalancer, EKS `nodeSelector`). Observability: optional **kube-prometheus-stack** Application (Helm chart in cluster, not Bitnami node pools). -For **LiteLLM + mock LLM + mock files API + Prometheus** with **Neon** (`DATABASE_URL` Secret) and **in-memory cache**, see **`../kubernetes/performance-eks/`**. +**Legacy path:** in-cluster Postgres + Redis + static Prometheus under [`legacy/kustomize/`](./legacy/kustomize/) (manual `kubectl kustomize` only; legacy Postgres uses **emptyDir**). -## LiteLLM on Kubernetes (EKS-friendly, Helm chart) +--- -`litellm-eks/` mirrors the repo root `docker-compose.yml` layout: single replica, bundled Postgres, `STORE_MODEL_IN_DB`, and Helm migration hooks for plain `helm install`. +### What runs where -### AWS EKS prerequisites +| Piece | Role | +|--------|------| +| **EKS** | Cluster from `eksctl-cluster.yaml` (or console). | +| **Helm** | Packages LiteLLM, **Postgres + Redis** subcharts, migration Job, mock via `extraResources`. | +| **Argo CD** | Watches Git and syncs `Application` manifests in `argocd/`. | -- `kubectl` context points at your cluster (`aws eks update-kubeconfig ...`). -- A **default StorageClass** (EKS often ships with `gp2` / `gp3` via EBS CSI driver) so the Bitnami Postgres chart can bind PVCs. -- For a public proxy URL, either **`kubectl port-forward`**, set **`service.type: LoadBalancer`** in `litellm-eks/values.yaml` (NLB/CLB), or add an **Ingress** + AWS Load Balancer Controller / ALB—see the main Helm chart `ingress` values. +--- -### Deploy with Helm only +## Runbook: Helm + Argo (recommended) -From the repository root: +| Step | Action | +|------|--------| +| **1–3** | AWS CLI, `kubectl`, `eksctl` — see [AWS CLI + kubectl](#aws-cli--kubectl-first-time-setup). | +| **4** | `eksctl create cluster -f deploy/performance/eksctl-cluster.yaml` | +| **5** | `aws eks update-kubeconfig --region us-east-1 --name litellm-performance` (or your `metadata.name` / region from `eksctl-cluster.yaml`) | +| **6** | **EBS CSI** — Bitnami Postgres (and often Redis) use PVCs. Install the [EKS EBS CSI add-on](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html) and ensure a default `StorageClass` exists. | +| **7** | **Install Argo CD** — `kubectl create namespace argocd` then `kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml` | +| **8** | **Register Git repo** if private (Argo **Settings → Repositories**). | +| **9** | **Apply Applications** (pin `targetRevision` in each file to your branch if needed): | ```bash -helm dependency update deploy/charts/litellm-helm -chmod +x deploy/performance/litellm-eks/install.sh -MASTER_KEY='sk-your-key' ./deploy/performance/litellm-eks/install.sh +kubectl apply -n argocd -f deploy/performance/argocd/application-litellm.yaml +kubectl apply -n argocd -f deploy/performance/argocd/application-kube-prometheus-stack.yaml ``` -Optional environment variables: `LITELLM_NAMESPACE`, `LITELLM_RELEASE`, `MASTER_KEY`. +| **10** | Argo UI → sync **litellm-helm**, then **kube-prometheus-stack** (large first sync). | +| **11** | **Postgres / Redis** — override defaults in `helm/litellm-values.yaml` (`postgresql.auth`, `redis.auth`) for non-dev clusters; re-sync after edits. | +| **12** | **Master key** — chart creates `litellm-masterkey` by default; override with `masterkeySecretName` + your Secret if required. | +| **13** | Verify: `kubectl -n performance get pods,svc` — `litellm` Service **LoadBalancer** on port **4000**. | -### Deploy with Kustomize + Helm (chart inflator) +--- -Kustomize renders the same chart; you then apply YAML (no Helm release stored in the cluster). +## Layout + +``` +deploy/performance/ + argocd/ + application-litellm.yaml + application-kube-prometheus-stack.yaml + helm/ + litellm-values.yaml + eksctl-cluster.yaml + legacy/ + dev-config.yaml # Proxy YAML for legacy Kustomize only + kustomize/ # Raw Postgres + Redis + Prometheus (optional) +``` + +--- + +## Helm values + +- **Source of truth:** `helm/litellm-values.yaml` → `proxy_config`. Legacy overlay uses `legacy/dev-config.yaml` only if you still run Kustomize. +- **Argo** must sync a `targetRevision` that contains this file (paths in `application-litellm.yaml` are relative to `deploy/charts/litellm-helm`). + +--- + +## Troubleshooting reference + +### Stuck Postgres `Pending` / PVC `FailedBinding` + +Install **EBS CSI** and a **StorageClass**; see [EKS EBS CSI](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html). + +### Stuck LiteLLM `Init:*` / old `wait-db` + +That was an **older** raw Deployment spec. This Helm chart does not use those init containers. + +### EBS CSI / IRSA / OIDC + +The EBS CSI controller must use **IRSA** (`eks.amazonaws.com/role-arn` on the ServiceAccount). Typical fixes: + +- Associate OIDC: `eksctl utils associate-iam-oidc-provider --cluster --region --approve` +- Create role + annotate SA: `eksctl create iamserviceaccount ... --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy` +- Flag: **`--override-existing-serviceaccounts`** (plural) +- CloudFormation stack **`ROLLBACK_COMPLETE`**: disable termination protection, delete stack, retry +- Set add-on role: `aws eks update-addon --cluster-name --addon-name aws-ebs-csi-driver --service-account-role-arn --resolve-conflicts OVERWRITE` + +## Legacy: Kustomize (Postgres + Redis + Prometheus) + +Create the namespace and Secret **first** (password must match `DATABASE_URL`): ```bash -helm dependency update deploy/charts/litellm-helm -kubectl kustomize deploy/performance/litellm-eks --enable-helm | kubectl apply -f - +kubectl apply -f deploy/performance/legacy/kustomize/namespace.yaml +kubectl create secret generic litellm-dotenv \ + --namespace performance \ + --from-literal=DATABASE_PASSWORD='choose-a-password' \ + --from-literal=DATABASE_URL='postgresql://litellm:choose-a-password@postgres:5432/litellm' \ + --from-literal=PROXY_MASTER_KEY='sk-your-admin-key' \ + --dry-run=client -o yaml | kubectl apply -f - ``` -To pin the proxy master key when using this path, add a local values file and reference it from `litellm-eks/kustomization.yaml` under `helmCharts[].additionalValuesFiles` (keep that file out of git). +Then apply the overlay (from repo root): -**Note:** `helmCharts[].version` in `kustomization.yaml` must match `version` in `deploy/charts/litellm-helm/Chart.yaml` after chart bumps. +```bash +kubectl kustomize deploy/performance/legacy/kustomize \ + --load-restrictor=LoadRestrictionsNone \ + | kubectl apply -f - +``` -### Prometheus +Postgres in this overlay is **emptyDir** (ephemeral). For PVC-backed DBs, use the **Helm** chart. -The compose file scrapes `litellm:4000`. In-cluster, point Prometheus at `http://-litellm..svc.cluster.local:4000` (default install: `http://litellm-litellm.litellm.svc.cluster.local:4000`). +--- + +## TODOs / next steps + +- [ ] **Pin Argo `targetRevision`** to a tag or SHA for reproducibility. +- [ ] **kube-prometheus-stack:** Add scrape config for LiteLLM metrics via chart values. +- [ ] **Secrets:** External Secrets / SSM for `postgresql.auth`, `redis.auth`, and `PROXY_MASTER_KEY` in production. + +--- + +## AWS CLI + kubectl (first-time setup) + +### 1. Install AWS CLI v2 + +Follow the [official install guide](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). Verify: + +```bash +aws --version # aws-cli/2.x +``` + +### 2. Credentials + +Use `aws configure`, SSO, or roles your org supports. Verify: + +```bash +aws sts get-caller-identity +``` + +### 3. kubectl + +Install a [kubectl version compatible with your EKS control plane](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html). + +### 4. eksctl + +Install [eksctl](https://eksctl.io/installation/). Create/delete cluster: + +```bash +eksctl create cluster -f deploy/performance/eksctl-cluster.yaml +# eksctl delete cluster -f deploy/performance/eksctl-cluster.yaml +``` + +--- + +## Argo CD UI (quick login) + +```bash +kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d && echo +kubectl -n argocd port-forward svc/argocd-server 8080:443 +``` + +Open **https://localhost:8080**, user **`admin`**. Docs: [Argo CD getting started](https://argo-cd.readthedocs.io/en/stable/getting_started/). + +--- + +## Teardown + +**Argo-managed:** delete the Applications (or let prune), then remove workloads/namespaces as needed. + +**Legacy Kustomize:** + +```bash +kubectl kustomize deploy/performance/legacy/kustomize \ + --load-restrictor=LoadRestrictionsNone \ + | kubectl delete -f - +``` + +### After removing `mock-files-api` from Git + +```bash +kubectl -n performance delete deploy,svc mock-files-api --ignore-not-found +kubectl -n performance delete configmap mock-files-api-code --ignore-not-found +``` diff --git a/deploy/performance/argocd/application-kube-prometheus-stack.yaml b/deploy/performance/argocd/application-kube-prometheus-stack.yaml new file mode 100644 index 00000000000..46ac2399173 --- /dev/null +++ b/deploy/performance/argocd/application-kube-prometheus-stack.yaml @@ -0,0 +1,29 @@ +# Argo CD Application: kube-prometheus-stack → namespace monitoring. +# +# Chart: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack +# +# Apply: +# kubectl apply -n argocd -f deploy/performance/argocd/application-kube-prometheus-stack.yaml +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kube-prometheus-stack + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/prometheus-community/helm-charts.git + targetRevision: main + path: charts/kube-prometheus-stack + helm: + releaseName: kube-prometheus-stack + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/deploy/performance/argocd/application-litellm.yaml b/deploy/performance/argocd/application-litellm.yaml new file mode 100644 index 00000000000..7bc290e03c1 --- /dev/null +++ b/deploy/performance/argocd/application-litellm.yaml @@ -0,0 +1,34 @@ +# Argo CD Application: LiteLLM Helm chart → namespace performance. +# +# Chart: deploy/charts/litellm-helm +# Values: deploy/performance/helm/litellm-values.yaml +# +# Install Argo CD (once): see deploy/performance/README.md. +# +# Apply: +# kubectl apply -n argocd -f deploy/performance/argocd/application-litellm.yaml +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: litellm-helm + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/BerriAI/litellm.git + targetRevision: litellm_eks_perf_bench_suite #temporary + path: deploy/charts/litellm-helm + helm: + releaseName: litellm + valueFiles: + - ../../performance/helm/litellm-values.yaml + destination: + server: https://kubernetes.default.svc + namespace: performance + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/deploy/performance/eksctl-cluster.yaml b/deploy/performance/eksctl-cluster.yaml new file mode 100644 index 00000000000..fd5ca0c18e3 --- /dev/null +++ b/deploy/performance/eksctl-cluster.yaml @@ -0,0 +1,30 @@ +# EKS cluster — single managed node group for LiteLLM + in-cluster Bitnami Postgres/Redis. +# Cluster name (metadata.name) is litellm-performance so you can create it while the old cluster +# (e.g. litellm-perf) is still deleting. Point kubectl at this cluster after create: +# aws eks update-kubeconfig --region us-east-1 --name litellm-performance +# +# Node label matches Helm nodeSelector in deploy/performance/helm/litellm-values.yaml (`workload: litellm`). +# +# Prometheus / Grafana: deploy via Argo (`argocd/application-kube-prometheus-stack.yaml`), not via eksctl. +# Bitnami charts only package Postgres/Redis; they do not define AWS node pools. +# +# Create: eksctl create cluster -f deploy/performance/eksctl-cluster.yaml +# Delete: eksctl delete cluster -f deploy/performance/eksctl-cluster.yaml +# +# To resize or add node groups later: `eksctl create nodegroup` / console, then drain old groups if needed. + +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig + +metadata: + name: litellm-performance + region: us-east-1 + +managedNodeGroups: + - name: litellm + instanceType: t3.xlarge + desiredCapacity: 1 + minSize: 1 + maxSize: 1 + labels: + workload: litellm diff --git a/deploy/performance/helm/litellm-values.yaml b/deploy/performance/helm/litellm-values.yaml new file mode 100644 index 00000000000..0b80c6800d0 --- /dev/null +++ b/deploy/performance/helm/litellm-values.yaml @@ -0,0 +1,90 @@ +# Helm values for EKS performance stack: LiteLLM + Bitnami Postgres + Bitnami Redis (chart subcharts). +# Argo CD: deploy/performance/argocd/application-litellm.yaml +# +# Override postgresql.auth and redis.auth passwords for non-dev clusters. + +replicaCount: 2 + +args: + - --config + - /etc/litellm/config.yaml + - --num_workers + - "2" + - --max_requests_before_restart + - "20000" + +service: + type: LoadBalancer + port: 4000 + +nodeSelector: + workload: litellm + +securityContext: + capabilities: + add: + - SYS_PTRACE + +resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "4" + memory: "4Gi" + +readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + +livenessProbe: + initialDelaySeconds: 45 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + +startupProbe: + failureThreshold: 30 + +envVars: + STORE_MODEL_IN_DB: "True" + ENV: "production" + LITELLM_ENVIRONMENT: "production" + +db: + deployStandalone: true + useExisting: false + +redis: + enabled: true + architecture: standalone + +migrationJob: + hooks: + argocd: + enabled: true + +proxy_config: + model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake-model + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + timeout: 40 + general_settings: + master_key: os.environ/PROXY_MASTER_KEY + health_check_details: false + database_url: os.environ/DATABASE_URL + litellm_settings: + drop_params: true + telemetry: false + callbacks: [] + cache: true + cache_params: + type: redis + supported_call_types: [] + +extraResources: [] diff --git a/deploy/kubernetes/performance-eks/dev-config.yaml b/deploy/performance/legacy/dev-config.yaml similarity index 58% rename from deploy/kubernetes/performance-eks/dev-config.yaml rename to deploy/performance/legacy/dev-config.yaml index 2833e3401b1..297d4469dc2 100644 --- a/deploy/kubernetes/performance-eks/dev-config.yaml +++ b/deploy/performance/legacy/dev-config.yaml @@ -1,5 +1,8 @@ -# Proxy config for performance-eks: Neon (DATABASE_URL from Secret) + in-memory cache (no Redis). -# DB URL and master key must come from env — see README (litellm-dotenv Secret). +# Proxy config for legacy Kustomize only (in-cluster Postgres + Redis). See kustomize/kustomization.yaml. +# Helm + Argo: use deploy/performance/helm/litellm-values.yaml → proxy_config. +# +# Legacy Secret litellm-dotenv: DATABASE_URL, DATABASE_PASSWORD (same value as in URL), PROXY_MASTER_KEY. +# Redis: litellm-static-env ConfigMap sets REDIS_HOST / REDIS_PORT for this overlay. model_list: - model_name: fake-openai-endpoint @@ -9,12 +12,6 @@ model_list: api_base: https://exampleopenaiendpoint-production.up.railway.app/ timeout: 40 - - model_name: fake-openai-gpt4 - litellm_params: - model: gpt-4 - api_key: fake-key - api_base: http://mock-llm-provider/llm/ - general_settings: master_key: os.environ/PROXY_MASTER_KEY health_check_details: False @@ -25,10 +22,10 @@ litellm_settings: json_logs: True drop_params: True telemetry: False - callbacks: ["prometheus"] + callbacks: [] public_routes: ["LiteLLMRoutes.public_routes", "/health/liveliness"] num_retries: 0 cache: True cache_params: - type: local + type: redis supported_call_types: [] diff --git a/deploy/performance/legacy/kustomize/kustomization.yaml b/deploy/performance/legacy/kustomize/kustomization.yaml new file mode 100644 index 00000000000..8697a663ee5 --- /dev/null +++ b/deploy/performance/legacy/kustomize/kustomization.yaml @@ -0,0 +1,29 @@ +# Legacy: in-cluster Postgres + Redis + Prometheus (no Helm). Prefer Helm + Argo (../..). +# +# Apply from repo root: +# kubectl kustomize deploy/performance/legacy/kustomize --load-restrictor=LoadRestrictionsNone | kubectl apply -f - +# +# Requires Secret litellm-dotenv: DATABASE_URL, DATABASE_PASSWORD, PROXY_MASTER_KEY — see parent README. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: performance + +resources: + - namespace.yaml + - postgres.yaml + - redis.yaml + - litellm-static-env.yaml + - prometheus.yaml + - litellm.yaml + +configMapGenerator: + - name: dev-config + files: + - config.yaml=../dev-config.yaml + - name: prometheus-config + files: + - prometheus.yml=../../../../prometheus.yml + +generatorOptions: + disableNameSuffixHash: true diff --git a/deploy/performance/legacy/kustomize/litellm-static-env.yaml b/deploy/performance/legacy/kustomize/litellm-static-env.yaml new file mode 100644 index 00000000000..ba453090e30 --- /dev/null +++ b/deploy/performance/legacy/kustomize/litellm-static-env.yaml @@ -0,0 +1,11 @@ +# Non-secret env. DATABASE_URL, DATABASE_PASSWORD, PROXY_MASTER_KEY: Secret litellm-dotenv (required). +apiVersion: v1 +kind: ConfigMap +metadata: + name: litellm-static-env +data: + STORE_MODEL_IN_DB: "True" + ENV: "production" + LITELLM_ENVIRONMENT: "production" + REDIS_HOST: "redis" + REDIS_PORT: "6379" \ No newline at end of file diff --git a/deploy/kubernetes/performance-eks/litellm.yaml b/deploy/performance/legacy/kustomize/litellm.yaml similarity index 97% rename from deploy/kubernetes/performance-eks/litellm.yaml rename to deploy/performance/legacy/kustomize/litellm.yaml index dfb1aacf282..06766f120ff 100644 --- a/deploy/kubernetes/performance-eks/litellm.yaml +++ b/deploy/performance/legacy/kustomize/litellm.yaml @@ -15,7 +15,7 @@ kind: Deployment metadata: name: litellm spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: litellm @@ -24,6 +24,8 @@ spec: labels: app: litellm spec: + nodeSelector: + workload: litellm containers: - name: litellm image: ghcr.io/berriai/litellm:v1.83.3-stable diff --git a/deploy/kubernetes/performance-eks/namespace.yaml b/deploy/performance/legacy/kustomize/namespace.yaml similarity index 63% rename from deploy/kubernetes/performance-eks/namespace.yaml rename to deploy/performance/legacy/kustomize/namespace.yaml index a7823df7d6c..fb922b2cf94 100644 --- a/deploy/kubernetes/performance-eks/namespace.yaml +++ b/deploy/performance/legacy/kustomize/namespace.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: performance-eks + name: performance diff --git a/deploy/performance/legacy/kustomize/postgres.yaml b/deploy/performance/legacy/kustomize/postgres.yaml new file mode 100644 index 00000000000..cabe9723007 --- /dev/null +++ b/deploy/performance/legacy/kustomize/postgres.yaml @@ -0,0 +1,54 @@ +# In-cluster Postgres for legacy overlay (not Neon). Ephemeral emptyDir — ok for perf/dev; use Helm for PVC-backed Bitnami Postgres. +apiVersion: v1 +kind: Service +metadata: + name: postgres +spec: + ports: + - port: 5432 + targetPort: 5432 + selector: + app: postgres +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:15-alpine + ports: + - containerPort: 5432 + env: + - name: POSTGRES_USER + value: litellm + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: litellm-dotenv + key: DATABASE_PASSWORD + - name: POSTGRES_DB + value: litellm + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: ["pg_isready", "-U", "litellm", "-d", "litellm"] + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: data + emptyDir: {} diff --git a/deploy/kubernetes/performance-eks/prometheus.yaml b/deploy/performance/legacy/kustomize/prometheus.yaml similarity index 95% rename from deploy/kubernetes/performance-eks/prometheus.yaml rename to deploy/performance/legacy/kustomize/prometheus.yaml index 12635375b08..685370994e6 100644 --- a/deploy/kubernetes/performance-eks/prometheus.yaml +++ b/deploy/performance/legacy/kustomize/prometheus.yaml @@ -23,6 +23,8 @@ spec: labels: app: prometheus spec: + nodeSelector: + workload: prometheus containers: - name: prometheus image: prom/prometheus diff --git a/deploy/performance/legacy/kustomize/redis.yaml b/deploy/performance/legacy/kustomize/redis.yaml new file mode 100644 index 00000000000..a39d9214424 --- /dev/null +++ b/deploy/performance/legacy/kustomize/redis.yaml @@ -0,0 +1,36 @@ +# In-cluster Redis for legacy overlay (matches chart behavior: REDIS_HOST / REDIS_PORT on LiteLLM pod). +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + ports: + - port: 6379 + targetPort: 6379 + selector: + app: redis +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 3 + periodSeconds: 5 diff --git a/dev_config.yaml b/dev_config.yaml index 64e3c14703e..bdb6be4a006 100644 --- a/dev_config.yaml +++ b/dev_config.yaml @@ -4,10 +4,65 @@ model_list: model: openai/fake-model api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ + timeout: 40 + + - model_name: fake-openai-gpt4 + litellm_params: + model: gpt-4 + api_key: fake-key + # Docker Compose service `mock-llm-provider` (host:8090). Requires network_mock: False for real HTTP. + api_base: http://mock-llm-provider/llm/ + + - model_name: fake-openai-gpt4 + litellm_params: + model: gpt-4 + api_key: fake-key + # Docker Compose service `mock-llm-provider` (host:8090). Requires network_mock: False for real HTTP. + api_base: http://mock-llm-provider/llm/ general_settings: master_key: sk-1234 + health_check_details: False + database_url: "postgresql://llmproxy:dbpassword9090@127.0.0.1:5432/litellm" # 👈 KEY CHANGE + # disable_spend_updates: True + # use_redis_transaction_buffer: True litellm_settings: + network_mock: False # must be false to call mock-llm-provider over the Docker network + json_logs: True drop_params: True telemetry: False + callbacks: ["prometheus"] + public_routes: ["LiteLLMRoutes.public_routes", "/health/liveliness"] + num_retries: 0 + + # Redis-backed cache (Compose service `redis`; env RATELIMIT_* in docker-compose.yml) + cache: True + cache_params: + # type: "local" + type: "redis" + host: os.environ/RATELIMIT_REDIS_ENDPOINT + port: os.environ/RATELIMIT_REDIS_PORT + # # Uncomment when Redis requires AUTH (set `REDIS_DEFAULT_PASSWORD` in `.env`): + # # password: os.environ/REDIS_DEFAULT_PASSWORD + # mode: "default_off" + # ttl: 60 + # max_connections: 500 + supported_call_types: [] # cache opt-in; Redis still used for router / rate limits when configured + +# Router Redis (same Compose Redis). Omit redis_password in `.env` if Redis has no auth. +router_settings: + redis_host: os.environ/RATELIMIT_REDIS_ENDPOINT + redis_port: os.environ/RATELIMIT_REDIS_PORT + # redis_password: os.environ/REDIS_DEFAULT_PASSWORD + +# Uncomment when `opinionated_api` is installed (iFood production shape): +# guardrails: +# - guardrail_name: "llm-firewall" +# litellm_params: +# guardrail: opinionated_api.custom_guardrails.llm_firewall.LLMFirewall +# mode: "during_call" +# - guardrail_name: "llm-firewall-post" +# litellm_params: +# guardrail: opinionated_api.custom_guardrails.llm_firewall.LLMFirewall +# mode: "post_call"