Update setup to use argo cd app. move older setup to legacy which needs to be deleted later

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -75,6 +75,8 @@ spec:
               value: {{ .Release.Name }}-postgresql
             - name: DATABASE_NAME
               value: litellm
+            - name: DATABASE_URL
+              value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
             {{- else if .Values.db.useExisting }}
             - name: DATABASE_USERNAME
               valueFrom:

deploy/kubernetes/performance-eks/eksctl-cluster.yaml

@@ -1,20 +0,0 @@
-# EKS cluster — single worker node. Safe to commit (no secrets).
-#
-# Create:  eksctl create cluster -f deploy/kubernetes/performance-eks/eksctl-cluster.yaml
-# Delete:  eksctl delete cluster -f deploy/kubernetes/performance-eks/eksctl-cluster.yaml
-#
-# To allow more nodes later: raise maxSize (and desiredCapacity), then scale or upgrade.
-
-apiVersion: eksctl.io/v1alpha5
-kind: ClusterConfig
-
-metadata:
-  name: litellm-perf
-  region: us-east-1
-
-managedNodeGroups:
-  - name: workers
-    instanceType: t3.xlarge
-    desiredCapacity: 1
-    minSize: 1
-    maxSize: 1

deploy/kubernetes/performance-eks/kustomization.yaml

@@ -1,27 +0,0 @@
-# Apply from repo root:
-#   kubectl kustomize deploy/kubernetes/performance-eks --load-restrictor=LoadRestrictionsNone | kubectl apply -f -
-#
-# Requires Secret litellm-dotenv with DATABASE_URL (Neon) and PROXY_MASTER_KEY — see README.
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: performance-eks
-
-resources:
-  - namespace.yaml
-  - litellm-static-env.yaml
-  - mock-llm.yaml
-  - mock-files-api.yaml
-  - prometheus.yaml
-  - litellm.yaml
-
-configMapGenerator:
-  - name: dev-config
-    files:
-      - config.yaml=dev-config.yaml
-  - name: prometheus-config
-    files:
-      - prometheus.yml=../../../prometheus.yml
-
-generatorOptions:
-  disableNameSuffixHash: true

deploy/kubernetes/performance-eks/litellm-static-env.yaml

@@ -1,9 +0,0 @@
-# Non-secret env. DATABASE_URL + PROXY_MASTER_KEY: set via Secret litellm-dotenv (required).
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: litellm-static-env
-data:
-  STORE_MODEL_IN_DB: "True"
-  ENV: "production"
-  LITELLM_ENVIRONMENT: "production"

deploy/kubernetes/performance-eks/mock-files-api.yaml

@@ -1,71 +0,0 @@
-# Minimal OpenAI-shaped mock + /health (compose used repo mock_files_api.py on a volume).
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: mock-files-api-code
-data:
-  mock_files_api.py: |
-    """Minimal files/OpenAI mock for k8s parity; extend to match your local mock_files_api.py."""
-    from fastapi import FastAPI
-
-    app = FastAPI()
-
-    @app.get("/health")
-    def health():
-        return {"status": "ok"}
-
-    @app.get("/v1/models")
-    def models():
-        return {"object": "list", "data": []}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: mock-files-api
-spec:
-  ports:
-    - port: 8001
-      targetPort: 8001
-  selector:
-    app: mock-files-api
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: mock-files-api
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mock-files-api
-  template:
-    metadata:
-      labels:
-        app: mock-files-api
-    spec:
-      containers:
-        - name: api
-          image: python:3.11-slim
-          workingDir: /app
-          ports:
-            - containerPort: 8001
-          command:
-            - /bin/sh
-            - -c
-            - pip install --no-cache-dir fastapi uvicorn && uvicorn mock_files_api:app --host 0.0.0.0 --port 8001
-          volumeMounts:
-            - name: code
-              mountPath: /app/mock_files_api.py
-              subPath: mock_files_api.py
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: 8001
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 12
-      volumes:
-        - name: code
-          configMap:
-            name: mock-files-api-code

deploy/kubernetes/performance-eks/mock-llm.yaml

@@ -1,36 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: mock-llm-provider
-spec:
-  ports:
-    - port: 80
-      targetPort: 80
-  selector:
-    app: mock-llm-provider
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: mock-llm-provider
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mock-llm-provider
-  template:
-    metadata:
-      labels:
-        app: mock-llm-provider
-    spec:
-      containers:
-        - name: httpbun
-          image: sharat87/httpbun:latest
-          ports:
-            - containerPort: 80
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 80
-            initialDelaySeconds: 3
-            periodSeconds: 5

deploy/performance/README.md

@@ -1,44 +1,182 @@
-# Performance-related deploy assets
+# Performance stack (EKS)
 
-## Full Kustomize stack on EKS (Neon + in-memory cache)
+**Primary path: Helm + Argo CD** — `deploy/charts/litellm-helm` with values in `helm/litellm-values.yaml` (bundled **Bitnami Postgres + Redis**, shared **Redis** cache, LoadBalancer, EKS `nodeSelector`). Observability: optional **kube-prometheus-stack** Application (Helm chart in cluster, not Bitnami node pools).
 
-For **LiteLLM + mock LLM + mock files API + Prometheus** with **Neon** (`DATABASE_URL` Secret) and **in-memory cache**, see **`../kubernetes/performance-eks/`**.
+**Legacy path:** in-cluster Postgres + Redis + static Prometheus under [`legacy/kustomize/`](./legacy/kustomize/) (manual `kubectl kustomize` only; legacy Postgres uses **emptyDir**).
 
-## LiteLLM on Kubernetes (EKS-friendly, Helm chart)
+---
 
-`litellm-eks/` mirrors the repo root `docker-compose.yml` layout: single replica, bundled Postgres, `STORE_MODEL_IN_DB`, and Helm migration hooks for plain `helm install`.
+### What runs where
 
-### AWS EKS prerequisites
+| Piece | Role |
+|--------|------|
+| **EKS** | Cluster from `eksctl-cluster.yaml` (or console). |
+| **Helm** | Packages LiteLLM, **Postgres + Redis** subcharts, migration Job, mock via `extraResources`. |
+| **Argo CD** | Watches Git and syncs `Application` manifests in `argocd/`. |
 
-- `kubectl` context points at your cluster (`aws eks update-kubeconfig ...`).
-- A **default StorageClass** (EKS often ships with `gp2` / `gp3` via EBS CSI driver) so the Bitnami Postgres chart can bind PVCs.
-- For a public proxy URL, either **`kubectl port-forward`**, set **`service.type: LoadBalancer`** in `litellm-eks/values.yaml` (NLB/CLB), or add an **Ingress** + AWS Load Balancer Controller / ALB—see the main Helm chart `ingress` values.
+---
 
-### Deploy with Helm only
+## Runbook: Helm + Argo (recommended)
 
-From the repository root:
+| Step | Action |
+|------|--------|
+| **1–3** | AWS CLI, `kubectl`, `eksctl` — see [AWS CLI + kubectl](#aws-cli--kubectl-first-time-setup). |
+| **4** | `eksctl create cluster -f deploy/performance/eksctl-cluster.yaml` |
+| **5** | `aws eks update-kubeconfig --region us-east-1 --name litellm-performance` (or your `metadata.name` / region from `eksctl-cluster.yaml`) |
+| **6** | **EBS CSI** — Bitnami Postgres (and often Redis) use PVCs. Install the [EKS EBS CSI add-on](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html) and ensure a default `StorageClass` exists. |
+| **7** | **Install Argo CD** — `kubectl create namespace argocd` then `kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml` |
+| **8** | **Register Git repo** if private (Argo **Settings → Repositories**). |
+| **9** | **Apply Applications** (pin `targetRevision` in each file to your branch if needed): |
 
 ```bash
-helm dependency update deploy/charts/litellm-helm
-chmod +x deploy/performance/litellm-eks/install.sh
-MASTER_KEY='sk-your-key' ./deploy/performance/litellm-eks/install.sh
+kubectl apply -n argocd -f deploy/performance/argocd/application-litellm.yaml
+kubectl apply -n argocd -f deploy/performance/argocd/application-kube-prometheus-stack.yaml
 ```
 
-Optional environment variables: `LITELLM_NAMESPACE`, `LITELLM_RELEASE`, `MASTER_KEY`.
+| **10** | Argo UI → sync **litellm-helm**, then **kube-prometheus-stack** (large first sync). |
+| **11** | **Postgres / Redis** — override defaults in `helm/litellm-values.yaml` (`postgresql.auth`, `redis.auth`) for non-dev clusters; re-sync after edits. |
+| **12** | **Master key** — chart creates `litellm-masterkey` by default; override with `masterkeySecretName` + your Secret if required. |
+| **13** | Verify: `kubectl -n performance get pods,svc` — `litellm` Service **LoadBalancer** on port **4000**. |
 
-### Deploy with Kustomize + Helm (chart inflator)
+---
 
-Kustomize renders the same chart; you then apply YAML (no Helm release stored in the cluster).
+## Layout
+
+```
+deploy/performance/
+  argocd/
+    application-litellm.yaml
+    application-kube-prometheus-stack.yaml
+  helm/
+    litellm-values.yaml
+  eksctl-cluster.yaml
+  legacy/
+    dev-config.yaml                 # Proxy YAML for legacy Kustomize only
+    kustomize/                      # Raw Postgres + Redis + Prometheus (optional)
+```
+
+---
+
+## Helm values
+
+- **Source of truth:** `helm/litellm-values.yaml` → `proxy_config`. Legacy overlay uses `legacy/dev-config.yaml` only if you still run Kustomize.
+- **Argo** must sync a `targetRevision` that contains this file (paths in `application-litellm.yaml` are relative to `deploy/charts/litellm-helm`).
+
+---
+
+## Troubleshooting reference
+
+### Stuck Postgres `Pending` / PVC `FailedBinding`
+
+Install **EBS CSI** and a **StorageClass**; see [EKS EBS CSI](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html).
+
+### Stuck LiteLLM `Init:*` / old `wait-db`
+
+That was an **older** raw Deployment spec. This Helm chart does not use those init containers.
+
+### EBS CSI / IRSA / OIDC
+
+The EBS CSI controller must use **IRSA** (`eks.amazonaws.com/role-arn` on the ServiceAccount). Typical fixes:
+
+- Associate OIDC: `eksctl utils associate-iam-oidc-provider --cluster <name> --region <region> --approve`
+- Create role + annotate SA: `eksctl create iamserviceaccount ... --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy`
+- Flag: **`--override-existing-serviceaccounts`** (plural)
+- CloudFormation stack **`ROLLBACK_COMPLETE`**: disable termination protection, delete stack, retry
+- Set add-on role: `aws eks update-addon --cluster-name <name> --addon-name aws-ebs-csi-driver --service-account-role-arn <role-arn> --resolve-conflicts OVERWRITE`
+
+## Legacy: Kustomize (Postgres + Redis + Prometheus)
+
+Create the namespace and Secret **first** (password must match `DATABASE_URL`):
 
 ```bash
-helm dependency update deploy/charts/litellm-helm
-kubectl kustomize deploy/performance/litellm-eks --enable-helm | kubectl apply -f -
+kubectl apply -f deploy/performance/legacy/kustomize/namespace.yaml
+kubectl create secret generic litellm-dotenv \
+  --namespace performance \
+  --from-literal=DATABASE_PASSWORD='choose-a-password' \
+  --from-literal=DATABASE_URL='postgresql://litellm:choose-a-password@postgres:5432/litellm' \
+  --from-literal=PROXY_MASTER_KEY='sk-your-admin-key' \
+  --dry-run=client -o yaml | kubectl apply -f -
 ```
 
-To pin the proxy master key when using this path, add a local values file and reference it from `litellm-eks/kustomization.yaml` under `helmCharts[].additionalValuesFiles` (keep that file out of git).
+Then apply the overlay (from repo root):
 
-**Note:** `helmCharts[].version` in `kustomization.yaml` must match `version` in `deploy/charts/litellm-helm/Chart.yaml` after chart bumps.
+```bash
+kubectl kustomize deploy/performance/legacy/kustomize \
+  --load-restrictor=LoadRestrictionsNone \
+  | kubectl apply -f -
+```
 
-### Prometheus
+Postgres in this overlay is **emptyDir** (ephemeral). For PVC-backed DBs, use the **Helm** chart.
 
-The compose file scrapes `litellm:4000`. In-cluster, point Prometheus at `http://<release>-litellm.<namespace>.svc.cluster.local:4000` (default install: `http://litellm-litellm.litellm.svc.cluster.local:4000`).
+---
+
+## TODOs / next steps
+
+- [ ] **Pin Argo `targetRevision`** to a tag or SHA for reproducibility.
+- [ ] **kube-prometheus-stack:** Add scrape config for LiteLLM metrics via chart values.
+- [ ] **Secrets:** External Secrets / SSM for `postgresql.auth`, `redis.auth`, and `PROXY_MASTER_KEY` in production.
+
+---
+
+## AWS CLI + kubectl (first-time setup)
+
+### 1. Install AWS CLI v2
+
+Follow the [official install guide](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). Verify:
+
+```bash
+aws --version   # aws-cli/2.x
+```
+
+### 2. Credentials
+
+Use `aws configure`, SSO, or roles your org supports. Verify:
+
+```bash
+aws sts get-caller-identity
+```
+
+### 3. kubectl
+
+Install a [kubectl version compatible with your EKS control plane](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html).
+
+### 4. eksctl
+
+Install [eksctl](https://eksctl.io/installation/). Create/delete cluster:
+
+```bash
+eksctl create cluster -f deploy/performance/eksctl-cluster.yaml
+# eksctl delete cluster -f deploy/performance/eksctl-cluster.yaml
+```
+
+---
+
+## Argo CD UI (quick login)
+
+```bash
+kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d && echo
+kubectl -n argocd port-forward svc/argocd-server 8080:443
+```
+
+Open **https://localhost:8080**, user **`admin`**. Docs: [Argo CD getting started](https://argo-cd.readthedocs.io/en/stable/getting_started/).
+
+---
+
+## Teardown
+
+**Argo-managed:** delete the Applications (or let prune), then remove workloads/namespaces as needed.
+
+**Legacy Kustomize:**
+
+```bash
+kubectl kustomize deploy/performance/legacy/kustomize \
+  --load-restrictor=LoadRestrictionsNone \
+  | kubectl delete -f -
+```
+
+### After removing `mock-files-api` from Git
+
+```bash
+kubectl -n performance delete deploy,svc mock-files-api --ignore-not-found
+kubectl -n performance delete configmap mock-files-api-code --ignore-not-found
+```

deploy/performance/argocd/application-kube-prometheus-stack.yaml

@@ -0,0 +1,29 @@
+# Argo CD Application: kube-prometheus-stack → namespace monitoring.
+#
+# Chart: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
+#
+# Apply:
+#   kubectl apply -n argocd -f deploy/performance/argocd/application-kube-prometheus-stack.yaml
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kube-prometheus-stack
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/prometheus-community/helm-charts.git
+    targetRevision: main
+    path: charts/kube-prometheus-stack
+    helm:
+      releaseName: kube-prometheus-stack
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

deploy/performance/argocd/application-litellm.yaml

@@ -0,0 +1,34 @@
+# Argo CD Application: LiteLLM Helm chart → namespace performance.
+#
+# Chart: deploy/charts/litellm-helm
+# Values: deploy/performance/helm/litellm-values.yaml
+#
+# Install Argo CD (once): see deploy/performance/README.md.
+#
+# Apply:
+#   kubectl apply -n argocd -f deploy/performance/argocd/application-litellm.yaml
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: litellm-helm
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/BerriAI/litellm.git
+    targetRevision: litellm_eks_perf_bench_suite #temporary
+    path: deploy/charts/litellm-helm
+    helm:
+      releaseName: litellm
+      valueFiles:
+        - ../../performance/helm/litellm-values.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: performance
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

deploy/performance/eksctl-cluster.yaml

@@ -0,0 +1,30 @@
+# EKS cluster — single managed node group for LiteLLM + in-cluster Bitnami Postgres/Redis.
+# Cluster name (metadata.name) is litellm-performance so you can create it while the old cluster
+# (e.g. litellm-perf) is still deleting. Point kubectl at this cluster after create:
+#   aws eks update-kubeconfig --region us-east-1 --name litellm-performance
+#
+# Node label matches Helm nodeSelector in deploy/performance/helm/litellm-values.yaml (`workload: litellm`).
+#
+# Prometheus / Grafana: deploy via Argo (`argocd/application-kube-prometheus-stack.yaml`), not via eksctl.
+# Bitnami charts only package Postgres/Redis; they do not define AWS node pools.
+#
+# Create:  eksctl create cluster -f deploy/performance/eksctl-cluster.yaml
+# Delete:  eksctl delete cluster -f deploy/performance/eksctl-cluster.yaml
+#
+# To resize or add node groups later: `eksctl create nodegroup` / console, then drain old groups if needed.
+
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+  name: litellm-performance
+  region: us-east-1
+
+managedNodeGroups:
+  - name: litellm
+    instanceType: t3.xlarge
+    desiredCapacity: 1
+    minSize: 1
+    maxSize: 1
+    labels:
+      workload: litellm

deploy/performance/helm/litellm-values.yaml

@@ -0,0 +1,90 @@
+# Helm values for EKS performance stack: LiteLLM + Bitnami Postgres + Bitnami Redis (chart subcharts).
+# Argo CD: deploy/performance/argocd/application-litellm.yaml
+#
+# Override postgresql.auth and redis.auth passwords for non-dev clusters.
+
+replicaCount: 2
+
+args:
+  - --config
+  - /etc/litellm/config.yaml
+  - --num_workers
+  - "2"
+  - --max_requests_before_restart
+  - "20000"
+
+service:
+  type: LoadBalancer
+  port: 4000
+
+nodeSelector:
+  workload: litellm
+
+securityContext:
+  capabilities:
+    add:
+      - SYS_PTRACE
+
+resources:
+  requests:
+    cpu: "500m"
+    memory: "1Gi"
+  limits:
+    cpu: "4"
+    memory: "4Gi"
+
+readinessProbe:
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  timeoutSeconds: 5
+  failureThreshold: 6
+
+livenessProbe:
+  initialDelaySeconds: 45
+  periodSeconds: 30
+  timeoutSeconds: 10
+  failureThreshold: 3
+
+startupProbe:
+  failureThreshold: 30
+
+envVars:
+  STORE_MODEL_IN_DB: "True"
+  ENV: "production"
+  LITELLM_ENVIRONMENT: "production"
+
+db:
+  deployStandalone: true
+  useExisting: false
+
+redis:
+  enabled: true
+  architecture: standalone
+
+migrationJob:
+  hooks:
+    argocd:
+      enabled: true
+
+proxy_config:
+  model_list:
+    - model_name: fake-openai-endpoint
+      litellm_params:
+        model: openai/fake-model
+        api_key: fake-key
+        api_base: https://exampleopenaiendpoint-production.up.railway.app/
+        timeout: 40
+  general_settings:
+    master_key: os.environ/PROXY_MASTER_KEY
+    health_check_details: false
+    database_url: os.environ/DATABASE_URL
+  litellm_settings:
+    drop_params: true
+    telemetry: false
+    callbacks: []
+    cache: true
+    cache_params:
+      type: redis
+      supported_call_types: []
+
+extraResources: []

deploy/performance/legacy/dev-config.yaml

@@ -1,5 +1,8 @@
-# Proxy config for performance-eks: Neon (DATABASE_URL from Secret) + in-memory cache (no Redis).
-# DB URL and master key must come from env — see README (litellm-dotenv Secret).
+# Proxy config for legacy Kustomize only (in-cluster Postgres + Redis). See kustomize/kustomization.yaml.
+# Helm + Argo: use deploy/performance/helm/litellm-values.yaml → proxy_config.
+#
+# Legacy Secret litellm-dotenv: DATABASE_URL, DATABASE_PASSWORD (same value as in URL), PROXY_MASTER_KEY.
+# Redis: litellm-static-env ConfigMap sets REDIS_HOST / REDIS_PORT for this overlay.
 
 model_list:
   - model_name: fake-openai-endpoint
@@ -9,12 +12,6 @@ model_list:
       api_base: https://exampleopenaiendpoint-production.up.railway.app/
       timeout: 40
 
-  - model_name: fake-openai-gpt4
-    litellm_params:
-      model: gpt-4
-      api_key: fake-key
-      api_base: http://mock-llm-provider/llm/
-
 general_settings:
   master_key: os.environ/PROXY_MASTER_KEY
   health_check_details: False
@@ -25,10 +22,10 @@ litellm_settings:
   json_logs: True
   drop_params: True
   telemetry: False
-  callbacks: ["prometheus"]
+  callbacks: []
   public_routes: ["LiteLLMRoutes.public_routes", "/health/liveliness"]
   num_retries: 0
   cache: True
   cache_params:
-    type: local
+    type: redis
     supported_call_types: []

deploy/performance/legacy/kustomize/kustomization.yaml

@@ -0,0 +1,29 @@
+# Legacy: in-cluster Postgres + Redis + Prometheus (no Helm). Prefer Helm + Argo (../..).
+#
+# Apply from repo root:
+#   kubectl kustomize deploy/performance/legacy/kustomize --load-restrictor=LoadRestrictionsNone | kubectl apply -f -
+#
+# Requires Secret litellm-dotenv: DATABASE_URL, DATABASE_PASSWORD, PROXY_MASTER_KEY — see parent README.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: performance
+
+resources:
+  - namespace.yaml
+  - postgres.yaml
+  - redis.yaml
+  - litellm-static-env.yaml
+  - prometheus.yaml
+  - litellm.yaml
+
+configMapGenerator:
+  - name: dev-config
+    files:
+      - config.yaml=../dev-config.yaml
+  - name: prometheus-config
+    files:
+      - prometheus.yml=../../../../prometheus.yml
+
+generatorOptions:
+  disableNameSuffixHash: true

deploy/performance/legacy/kustomize/litellm-static-env.yaml

@@ -0,0 +1,11 @@
+# Non-secret env. DATABASE_URL, DATABASE_PASSWORD, PROXY_MASTER_KEY: Secret litellm-dotenv (required).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: litellm-static-env
+data:
+  STORE_MODEL_IN_DB: "True"
+  ENV: "production"
+  LITELLM_ENVIRONMENT: "production"
+  REDIS_HOST: "redis"
+  REDIS_PORT: "6379"

deploy/performance/legacy/kustomize/litellm.yaml

@@ -15,7 +15,7 @@ kind: Deployment
 metadata:
   name: litellm
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: litellm
@@ -24,6 +24,8 @@ spec:
       labels:
         app: litellm
     spec:
+      nodeSelector:
+        workload: litellm
       containers:
         - name: litellm
           image: ghcr.io/berriai/litellm:v1.83.3-stable

deploy/performance/legacy/kustomize/namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: performance-eks
+  name: performance

deploy/performance/legacy/kustomize/postgres.yaml

@@ -0,0 +1,54 @@
+# In-cluster Postgres for legacy overlay (not Neon). Ephemeral emptyDir — ok for perf/dev; use Helm for PVC-backed Bitnami Postgres.
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  ports:
+    - port: 5432
+      targetPort: 5432
+  selector:
+    app: postgres
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:15-alpine
+          ports:
+            - containerPort: 5432
+          env:
+            - name: POSTGRES_USER
+              value: litellm
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-dotenv
+                  key: DATABASE_PASSWORD
+            - name: POSTGRES_DB
+              value: litellm
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/postgresql/data
+          readinessProbe:
+            exec:
+              command: ["pg_isready", "-U", "litellm", "-d", "litellm"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+      volumes:
+        - name: data
+          emptyDir: {}

deploy/performance/legacy/kustomize/prometheus.yaml

@@ -23,6 +23,8 @@ spec:
       labels:
         app: prometheus
     spec:
+      nodeSelector:
+        workload: prometheus
       containers:
         - name: prometheus
           image: prom/prometheus

deploy/performance/legacy/kustomize/redis.yaml

@@ -0,0 +1,36 @@
+# In-cluster Redis for legacy overlay (matches chart behavior: REDIS_HOST / REDIS_PORT on LiteLLM pod).
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+spec:
+  ports:
+    - port: 6379
+      targetPort: 6379
+  selector:
+    app: redis
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:7-alpine
+          ports:
+            - containerPort: 6379
+          readinessProbe:
+            exec:
+              command: ["redis-cli", "ping"]
+            initialDelaySeconds: 3
+            periodSeconds: 5

dev_config.yaml

@@ -4,10 +4,65 @@ model_list:
       model: openai/fake-model
       api_key: fake-key
       api_base: https://exampleopenaiendpoint-production.up.railway.app/
+      timeout: 40
+
+  - model_name: fake-openai-gpt4
+    litellm_params:
+      model: gpt-4
+      api_key: fake-key
+      # Docker Compose service `mock-llm-provider` (host:8090). Requires network_mock: False for real HTTP.
+      api_base: http://mock-llm-provider/llm/
+
+  - model_name: fake-openai-gpt4
+    litellm_params:
+      model: gpt-4
+      api_key: fake-key
+      # Docker Compose service `mock-llm-provider` (host:8090). Requires network_mock: False for real HTTP.
+      api_base: http://mock-llm-provider/llm/
 
 general_settings:
   master_key: sk-1234
+  health_check_details: False
+  database_url: "postgresql://llmproxy:dbpassword9090@127.0.0.1:5432/litellm" # 👈 KEY CHANGE
+  # disable_spend_updates: True
+  # use_redis_transaction_buffer: True
 
 litellm_settings:
+  network_mock: False # must be false to call mock-llm-provider over the Docker network
+  json_logs: True
   drop_params: True
   telemetry: False
+  callbacks: ["prometheus"]
+  public_routes: ["LiteLLMRoutes.public_routes", "/health/liveliness"]
+  num_retries: 0
+
+  # Redis-backed cache (Compose service `redis`; env RATELIMIT_* in docker-compose.yml)
+  cache: True
+  cache_params:
+    # type: "local"
+    type: "redis"
+    host: os.environ/RATELIMIT_REDIS_ENDPOINT
+    port: os.environ/RATELIMIT_REDIS_PORT
+    # # Uncomment when Redis requires AUTH (set `REDIS_DEFAULT_PASSWORD` in `.env`):
+    # # password: os.environ/REDIS_DEFAULT_PASSWORD
+    # mode: "default_off"
+    # ttl: 60
+    # max_connections: 500
+    supported_call_types: [] # cache opt-in; Redis still used for router / rate limits when configured
+
+# Router Redis (same Compose Redis). Omit redis_password in `.env` if Redis has no auth.
+router_settings:
+  redis_host: os.environ/RATELIMIT_REDIS_ENDPOINT
+  redis_port: os.environ/RATELIMIT_REDIS_PORT
+  # redis_password: os.environ/REDIS_DEFAULT_PASSWORD
+
+# Uncomment when `opinionated_api` is installed (iFood production shape):
+# guardrails:
+#   - guardrail_name: "llm-firewall"
+#     litellm_params:
+#       guardrail: opinionated_api.custom_guardrails.llm_firewall.LLMFirewall
+#       mode: "during_call"
+#   - guardrail_name: "llm-firewall-post"
+#     litellm_params:
+#       guardrail: opinionated_api.custom_guardrails.llm_firewall.LLMFirewall
+#       mode: "post_call"