feat(deploy): add autorouter sandbox gateway that proxies to the upstream sandbox

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/workflows/deploy-autorouter-sandbox.yml

@@ -0,0 +1,52 @@
+name: Deploy autorouter sandbox
+
+on:
+  push:
+    branches:
+      - litellm_autorouter_sandbox_deploy
+  workflow_dispatch:
+
+concurrency:
+  group: deploy-autorouter-sandbox
+  cancel-in-progress: true
+
+env:
+  IMAGE: ghcr.io/${{ github.repository_owner }}/litellm-autorouter-sandbox
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+      - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: deploy/autorouter-sandbox/Dockerfile
+          push: true
+          tags: |
+            ${{ env.IMAGE }}:${{ github.sha }}
+            ${{ env.IMAGE }}:latest
+
+      - if: ${{ secrets.AUTOROUTER_SANDBOX_KUBECONFIG_B64 != '' }}
+        uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v5.1.0
+
+      - if: ${{ secrets.AUTOROUTER_SANDBOX_KUBECONFIG_B64 != '' }}
+        env:
+          KUBECONFIG_B64: ${{ secrets.AUTOROUTER_SANDBOX_KUBECONFIG_B64 }}
+          NAMESPACE: ${{ vars.AUTOROUTER_SANDBOX_NAMESPACE || 'default' }}
+        run: |
+          echo "$KUBECONFIG_B64" | base64 -d > "$RUNNER_TEMP/kubeconfig"
+          export KUBECONFIG="$RUNNER_TEMP/kubeconfig"
+          kubectl -n "$NAMESPACE" apply -f deploy/autorouter-sandbox/k8s.yaml
+          kubectl -n "$NAMESPACE" set image deployment/litellm-autorouter-sandbox litellm="$IMAGE:$GITHUB_SHA"
+          kubectl -n "$NAMESPACE" rollout status deployment/litellm-autorouter-sandbox --timeout=5m

deploy/autorouter-sandbox/.env.example

@@ -0,0 +1,5 @@
+UPSTREAM_LITELLM_BASE_URL=https://gateway.litellm-sandbox.ai
+UPSTREAM_LITELLM_API_KEY=sk-...   # an admin (or wide-access) virtual key on the upstream gateway
+LITELLM_MASTER_KEY=sk-...         # master key clients use against THIS instance
+# Optional. Without it there are no virtual keys / spend logs, only the master key.
+# DATABASE_URL=postgresql://user:pass@host:5432/litellm

deploy/autorouter-sandbox/Dockerfile

@@ -0,0 +1,13 @@
+ARG BASE_IMAGE=ghcr.io/berriai/litellm:main-latest
+FROM ${BASE_IMAGE}
+
+ARG SITE_PACKAGES=/app/.venv/lib/python3.13/site-packages
+COPY litellm/ ${SITE_PACKAGES}/litellm/
+COPY enterprise/litellm_enterprise/ ${SITE_PACKAGES}/litellm_enterprise/
+COPY enterprise/ /app/enterprise/
+COPY litellm-proxy-extras/litellm_proxy_extras/ ${SITE_PACKAGES}/litellm_proxy_extras/
+COPY litellm-proxy-extras/ /app/litellm-proxy-extras/
+COPY schema.prisma /app/schema.prisma
+COPY deploy/autorouter-sandbox/config.yaml /app/config.yaml
+
+CMD ["--config", "/app/config.yaml", "--port", "4000"]

deploy/autorouter-sandbox/README.md

@@ -0,0 +1,25 @@
+# Autorouter sandbox gateway
+
+A LiteLLM deployment built from this branch that you can push to and redeploy in about a minute, for iterating on the auto router without touching the real sandbox or prod
+
+Every model that is not defined in `config.yaml` is forwarded to the upstream gateway (`UPSTREAM_LITELLM_BASE_URL`, i.e. gateway.litellm-sandbox.ai) through `litellm_proxy/*`, so this instance inherits all upstream models and provider credentials without copying any keys. Auto routers defined here (`moe-router`) pick between those upstream models and the router code running is whatever is on this branch
+
+## Image
+
+`Dockerfile` overlays this branch's Python source onto the published `ghcr.io/berriai/litellm:main-latest` image instead of rebuilding deps and the UI, so a build takes seconds rather than ~15 minutes. If the branch adds a new dependency, switch `BASE_IMAGE` to an image built from the root `Dockerfile`
+
+## Run
+
+```bash
+cp deploy/autorouter-sandbox/.env.example deploy/autorouter-sandbox/.env   # fill in
+docker compose -f deploy/autorouter-sandbox/docker-compose.yml up --build -d
+curl localhost:4000/v1/chat/completions -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"moe-router","messages":[{"role":"user","content":"hi"}]}' -i | grep x-litellm-model-name
+```
+
+Without Docker: `uv run --no-sync litellm --config deploy/autorouter-sandbox/config.yaml --port 4000`
+
+## Deploy
+
+`.github/workflows/deploy-autorouter-sandbox.yml` builds and pushes `ghcr.io/<owner>/litellm-autorouter-sandbox:{sha,latest}` on every push to `litellm_autorouter_sandbox_deploy`. If the repo secret `AUTOROUTER_SANDBOX_KUBECONFIG_B64` is set (base64 kubeconfig; optional repo variable `AUTOROUTER_SANDBOX_NAMESPACE`), it also applies `k8s.yaml` and rolls the deployment to the new image. Create the env secret once with the command at the top of `k8s.yaml`

deploy/autorouter-sandbox/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  litellm:
+    image: ${LITELLM_IMAGE:-ghcr.io/berriai/litellm-autorouter-sandbox:latest}
+    build:
+      context: ../..
+      dockerfile: deploy/autorouter-sandbox/Dockerfile
+    ports:
+      - "4000:4000"
+    env_file:
+      - .env
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"]
+      interval: 30s
+      timeout: 5s
+      retries: 3

deploy/autorouter-sandbox/k8s.yaml

@@ -0,0 +1,60 @@
+# kubectl create secret generic litellm-autorouter-sandbox-env \
+#   --from-literal=UPSTREAM_LITELLM_BASE_URL=https://gateway.litellm-sandbox.ai \
+#   --from-literal=UPSTREAM_LITELLM_API_KEY=sk-... \
+#   --from-literal=LITELLM_MASTER_KEY=sk-...
+# kubectl apply -f k8s.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: litellm-autorouter-sandbox
+  labels:
+    app: litellm-autorouter-sandbox
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: litellm-autorouter-sandbox
+  template:
+    metadata:
+      labels:
+        app: litellm-autorouter-sandbox
+    spec:
+      containers:
+        - name: litellm
+          image: ghcr.io/berriai/litellm-autorouter-sandbox:latest
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 4000
+          envFrom:
+            - secretRef:
+                name: litellm-autorouter-sandbox-env
+          readinessProbe:
+            httpGet:
+              path: /health/readiness
+              port: 4000
+            initialDelaySeconds: 15
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health/liveliness
+              port: 4000
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              memory: 2Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: litellm-autorouter-sandbox
+spec:
+  type: LoadBalancer
+  selector:
+    app: litellm-autorouter-sandbox
+  ports:
+    - port: 80
+      targetPort: 4000