From 4b269c0ac4a95c6638a473ffa22bf2972cfcf7eb Mon Sep 17 00:00:00 2001
From: harish-berri <harish@berri.ai>
Date: Wed, 22 Apr 2026 01:27:06 +0000
Subject: [PATCH] Update dev_config.yaml and Helm chart dependencies for
 performance-eks deployment

- Added new configuration options in dev_config.yaml, including health check details and database URL.
- Updated PostgreSQL and Redis dependencies in Chart.lock to newer versions (PostgreSQL 18.6.1, Redis 25.3.12).
- Introduced new Kubernetes deployment files for performance-eks, including configurations for LiteLLM, mock LLM provider, and Prometheus.
- Added README.md for performance-related deployment assets.
---
 .../performance-eks/dev-config.yaml           | 34 ++++++++
 .../performance-eks/eksctl-cluster.yaml       | 20 +++++
 .../performance-eks/kustomization.yaml        | 27 ++++++
 .../performance-eks/litellm-static-env.yaml   |  9 ++
 .../kubernetes/performance-eks/litellm.yaml   | 84 +++++++++++++++++++
 .../performance-eks/mock-files-api.yaml       | 71 ++++++++++++++++
 .../kubernetes/performance-eks/mock-llm.yaml  | 36 ++++++++
 .../kubernetes/performance-eks/namespace.yaml |  4 +
 .../performance-eks/prometheus.yaml           | 47 +++++++++++
 deploy/performance/README.md                  | 44 ++++++++++
 10 files changed, 376 insertions(+)
 create mode 100644 deploy/kubernetes/performance-eks/dev-config.yaml
 create mode 100644 deploy/kubernetes/performance-eks/eksctl-cluster.yaml
 create mode 100644 deploy/kubernetes/performance-eks/kustomization.yaml
 create mode 100644 deploy/kubernetes/performance-eks/litellm-static-env.yaml
 create mode 100644 deploy/kubernetes/performance-eks/litellm.yaml
 create mode 100644 deploy/kubernetes/performance-eks/mock-files-api.yaml
 create mode 100644 deploy/kubernetes/performance-eks/mock-llm.yaml
 create mode 100644 deploy/kubernetes/performance-eks/namespace.yaml
 create mode 100644 deploy/kubernetes/performance-eks/prometheus.yaml
 create mode 100644 deploy/performance/README.md

diff --git a/deploy/kubernetes/performance-eks/dev-config.yaml b/deploy/kubernetes/performance-eks/dev-config.yaml
new file mode 100644
index 00000000000..2833e3401b1
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/dev-config.yaml
@@ -0,0 +1,34 @@
+# Proxy config for performance-eks: Neon (DATABASE_URL from Secret) + in-memory cache (no Redis).
+# DB URL and master key must come from env — see README (litellm-dotenv Secret).
+
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+      timeout: 40
+
+  - model_name: fake-openai-gpt4
+    litellm_params:
+      model: gpt-4
+      api_key: fake-key
+      api_base: http://mock-llm-provider/llm/
+
+general_settings:
+  master_key: os.environ/PROXY_MASTER_KEY
+  health_check_details: False
+  database_url: os.environ/DATABASE_URL
+
+litellm_settings:
+  network_mock: False
+  json_logs: True
+  drop_params: True
+  telemetry: False
+  callbacks: ["prometheus"]
+  public_routes: ["LiteLLMRoutes.public_routes", "/health/liveliness"]
+  num_retries: 0
+  cache: True
+  cache_params:
+    type: local
+    supported_call_types: []
diff --git a/deploy/kubernetes/performance-eks/eksctl-cluster.yaml b/deploy/kubernetes/performance-eks/eksctl-cluster.yaml
new file mode 100644
index 00000000000..9c74ee39a05
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/eksctl-cluster.yaml
@@ -0,0 +1,20 @@
+# EKS cluster — single worker node. Safe to commit (no secrets).
+#
+# Create:  eksctl create cluster -f deploy/kubernetes/performance-eks/eksctl-cluster.yaml
+# Delete:  eksctl delete cluster -f deploy/kubernetes/performance-eks/eksctl-cluster.yaml
+#
+# To allow more nodes later: raise maxSize (and desiredCapacity), then scale or upgrade.
+
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+  name: litellm-perf
+  region: us-east-1
+
+managedNodeGroups:
+  - name: workers
+    instanceType: t3.xlarge
+    desiredCapacity: 1
+    minSize: 1
+    maxSize: 1
diff --git a/deploy/kubernetes/performance-eks/kustomization.yaml b/deploy/kubernetes/performance-eks/kustomization.yaml
new file mode 100644
index 00000000000..590e33f9d7f
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/kustomization.yaml
@@ -0,0 +1,27 @@
+# Apply from repo root:
+#   kubectl kustomize deploy/kubernetes/performance-eks --load-restrictor=LoadRestrictionsNone | kubectl apply -f -
+#
+# Requires Secret litellm-dotenv with DATABASE_URL (Neon) and PROXY_MASTER_KEY — see README.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: performance-eks
+
+resources:
+  - namespace.yaml
+  - litellm-static-env.yaml
+  - mock-llm.yaml
+  - mock-files-api.yaml
+  - prometheus.yaml
+  - litellm.yaml
+
+configMapGenerator:
+  - name: dev-config
+    files:
+      - config.yaml=dev-config.yaml
+  - name: prometheus-config
+    files:
+      - prometheus.yml=../../../prometheus.yml
+
+generatorOptions:
+  disableNameSuffixHash: true
diff --git a/deploy/kubernetes/performance-eks/litellm-static-env.yaml b/deploy/kubernetes/performance-eks/litellm-static-env.yaml
new file mode 100644
index 00000000000..15360ba2efb
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/litellm-static-env.yaml
@@ -0,0 +1,9 @@
+# Non-secret env. DATABASE_URL + PROXY_MASTER_KEY: set via Secret litellm-dotenv (required).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: litellm-static-env
+data:
+  STORE_MODEL_IN_DB: "True"
+  ENV: "production"
+  LITELLM_ENVIRONMENT: "production"
\ No newline at end of file
diff --git a/deploy/kubernetes/performance-eks/litellm.yaml b/deploy/kubernetes/performance-eks/litellm.yaml
new file mode 100644
index 00000000000..dfb1aacf282
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/litellm.yaml
@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: litellm
+spec:
+  type: LoadBalancer
+  ports:
+    - port: 4000
+      targetPort: 4000
+  selector:
+    app: litellm
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: litellm
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: litellm
+  template:
+    metadata:
+      labels:
+        app: litellm
+    spec:
+      containers:
+        - name: litellm
+          image: ghcr.io/berriai/litellm:v1.83.3-stable
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--host"
+            - "0.0.0.0"
+            - "--config"
+            - "/app/config.yaml"
+            - "--port"
+            - "4000"
+            - "--num_workers"
+            - "2"
+            - "--max_requests_before_restart"
+            - "20000"
+          ports:
+            - containerPort: 4000
+          envFrom:
+            - configMapRef:
+                name: litellm-static-env
+            - secretRef:
+                name: litellm-dotenv
+          volumeMounts:
+            - name: config
+              mountPath: /app/config.yaml
+              subPath: config.yaml
+              readOnly: true
+          securityContext:
+            capabilities:
+              add:
+                - SYS_PTRACE
+          readinessProbe:
+            httpGet:
+              path: /health/readiness
+              port: 4000
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /health/liveliness
+              port: 4000
+            initialDelaySeconds: 45
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 3
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"
+            limits:
+              cpu: "4"
+              memory: "4Gi"
+      volumes:
+        - name: config
+          configMap:
+            name: dev-config
diff --git a/deploy/kubernetes/performance-eks/mock-files-api.yaml b/deploy/kubernetes/performance-eks/mock-files-api.yaml
new file mode 100644
index 00000000000..81f7ce8daea
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/mock-files-api.yaml
@@ -0,0 +1,71 @@
+# Minimal OpenAI-shaped mock + /health (compose used repo mock_files_api.py on a volume).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mock-files-api-code
+data:
+  mock_files_api.py: |
+    """Minimal files/OpenAI mock for k8s parity; extend to match your local mock_files_api.py."""
+    from fastapi import FastAPI
+
+    app = FastAPI()
+
+    @app.get("/health")
+    def health():
+        return {"status": "ok"}
+
+    @app.get("/v1/models")
+    def models():
+        return {"object": "list", "data": []}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-files-api
+spec:
+  ports:
+    - port: 8001
+      targetPort: 8001
+  selector:
+    app: mock-files-api
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mock-files-api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mock-files-api
+  template:
+    metadata:
+      labels:
+        app: mock-files-api
+    spec:
+      containers:
+        - name: api
+          image: python:3.11-slim
+          workingDir: /app
+          ports:
+            - containerPort: 8001
+          command:
+            - /bin/sh
+            - -c
+            - pip install --no-cache-dir fastapi uvicorn && uvicorn mock_files_api:app --host 0.0.0.0 --port 8001
+          volumeMounts:
+            - name: code
+              mountPath: /app/mock_files_api.py
+              subPath: mock_files_api.py
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8001
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 12
+      volumes:
+        - name: code
+          configMap:
+            name: mock-files-api-code
diff --git a/deploy/kubernetes/performance-eks/mock-llm.yaml b/deploy/kubernetes/performance-eks/mock-llm.yaml
new file mode 100644
index 00000000000..580302a3ddf
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/mock-llm.yaml
@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mock-llm-provider
+spec:
+  ports:
+    - port: 80
+      targetPort: 80
+  selector:
+    app: mock-llm-provider
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mock-llm-provider
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mock-llm-provider
+  template:
+    metadata:
+      labels:
+        app: mock-llm-provider
+    spec:
+      containers:
+        - name: httpbun
+          image: sharat87/httpbun:latest
+          ports:
+            - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            initialDelaySeconds: 3
+            periodSeconds: 5
diff --git a/deploy/kubernetes/performance-eks/namespace.yaml b/deploy/kubernetes/performance-eks/namespace.yaml
new file mode 100644
index 00000000000..a7823df7d6c
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: performance-eks
diff --git a/deploy/kubernetes/performance-eks/prometheus.yaml b/deploy/kubernetes/performance-eks/prometheus.yaml
new file mode 100644
index 00000000000..12635375b08
--- /dev/null
+++ b/deploy/kubernetes/performance-eks/prometheus.yaml
@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+spec:
+  ports:
+    - port: 9090
+      targetPort: 9090
+  selector:
+    app: prometheus
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      containers:
+        - name: prometheus
+          image: prom/prometheus
+          args:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention.time=15d
+          ports:
+            - containerPort: 9090
+          volumeMounts:
+            - name: cfg
+              mountPath: /etc/prometheus
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+            initialDelaySeconds: 5
+            periodSeconds: 5
+      volumes:
+        - name: cfg
+          configMap:
+            name: prometheus-config
diff --git a/deploy/performance/README.md b/deploy/performance/README.md
new file mode 100644
index 00000000000..0165e7386d1
--- /dev/null
+++ b/deploy/performance/README.md
@@ -0,0 +1,44 @@
+# Performance-related deploy assets
+
+## Full Kustomize stack on EKS (Neon + in-memory cache)
+
+For **LiteLLM + mock LLM + mock files API + Prometheus** with **Neon** (`DATABASE_URL` Secret) and **in-memory cache**, see **`../kubernetes/performance-eks/`**.
+
+## LiteLLM on Kubernetes (EKS-friendly, Helm chart)
+
+`litellm-eks/` mirrors the repo root `docker-compose.yml` layout: single replica, bundled Postgres, `STORE_MODEL_IN_DB`, and Helm migration hooks for plain `helm install`.
+
+### AWS EKS prerequisites
+
+- `kubectl` context points at your cluster (`aws eks update-kubeconfig ...`).
+- A **default StorageClass** (EKS often ships with `gp2` / `gp3` via EBS CSI driver) so the Bitnami Postgres chart can bind PVCs.
+- For a public proxy URL, either **`kubectl port-forward`**, set **`service.type: LoadBalancer`** in `litellm-eks/values.yaml` (NLB/CLB), or add an **Ingress** + AWS Load Balancer Controller / ALB—see the main Helm chart `ingress` values.
+
+### Deploy with Helm only
+
+From the repository root:
+
+```bash
+helm dependency update deploy/charts/litellm-helm
+chmod +x deploy/performance/litellm-eks/install.sh
+MASTER_KEY='sk-your-key' ./deploy/performance/litellm-eks/install.sh
+```
+
+Optional environment variables: `LITELLM_NAMESPACE`, `LITELLM_RELEASE`, `MASTER_KEY`.
+
+### Deploy with Kustomize + Helm (chart inflator)
+
+Kustomize renders the same chart; you then apply YAML (no Helm release stored in the cluster).
+
+```bash
+helm dependency update deploy/charts/litellm-helm
+kubectl kustomize deploy/performance/litellm-eks --enable-helm | kubectl apply -f -
+```
+
+To pin the proxy master key when using this path, add a local values file and reference it from `litellm-eks/kustomization.yaml` under `helmCharts[].additionalValuesFiles` (keep that file out of git).
+
+**Note:** `helmCharts[].version` in `kustomization.yaml` must match `version` in `deploy/charts/litellm-helm/Chart.yaml` after chart bumps.
+
+### Prometheus
+
+The compose file scrapes `litellm:4000`. In-cluster, point Prometheus at `http://<release>-litellm.<namespace>.svc.cluster.local:4000` (default install: `http://litellm-litellm.litellm.svc.cluster.local:4000`).