diff --git a/.github/workflows/deploy-autorouter-sandbox.yml b/.github/workflows/deploy-autorouter-sandbox.yml new file mode 100644 index 00000000000..99347e20c67 --- /dev/null +++ b/.github/workflows/deploy-autorouter-sandbox.yml @@ -0,0 +1,52 @@ +name: Deploy autorouter sandbox + +on: + push: + branches: + - litellm_autorouter_sandbox_deploy + workflow_dispatch: + +concurrency: + group: deploy-autorouter-sandbox + cancel-in-progress: true + +env: + IMAGE: ghcr.io/${{ github.repository_owner }}/litellm-autorouter-sandbox + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: deploy/autorouter-sandbox/Dockerfile + push: true + tags: | + ${{ env.IMAGE }}:${{ github.sha }} + ${{ env.IMAGE }}:latest + + - if: ${{ secrets.AUTOROUTER_SANDBOX_KUBECONFIG_B64 != '' }} + uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v5.1.0 + + - if: ${{ secrets.AUTOROUTER_SANDBOX_KUBECONFIG_B64 != '' }} + env: + KUBECONFIG_B64: ${{ secrets.AUTOROUTER_SANDBOX_KUBECONFIG_B64 }} + NAMESPACE: ${{ vars.AUTOROUTER_SANDBOX_NAMESPACE || 'default' }} + run: | + echo "$KUBECONFIG_B64" | base64 -d > "$RUNNER_TEMP/kubeconfig" + export KUBECONFIG="$RUNNER_TEMP/kubeconfig" + kubectl -n "$NAMESPACE" apply -f deploy/autorouter-sandbox/k8s.yaml + kubectl -n "$NAMESPACE" set image deployment/litellm-autorouter-sandbox litellm="$IMAGE:$GITHUB_SHA" + kubectl -n "$NAMESPACE" rollout status deployment/litellm-autorouter-sandbox --timeout=5m diff --git a/deploy/autorouter-sandbox/.env.example b/deploy/autorouter-sandbox/.env.example new file mode 100644 index 00000000000..13d6b4308ae --- /dev/null +++ b/deploy/autorouter-sandbox/.env.example @@ -0,0 +1,5 @@ +UPSTREAM_LITELLM_BASE_URL=https://gateway.litellm-sandbox.ai +UPSTREAM_LITELLM_API_KEY=sk-... # an admin (or wide-access) virtual key on the upstream gateway +LITELLM_MASTER_KEY=sk-... # master key clients use against THIS instance +# Optional. Without it there are no virtual keys / spend logs, only the master key. +# DATABASE_URL=postgresql://user:pass@host:5432/litellm diff --git a/deploy/autorouter-sandbox/Dockerfile b/deploy/autorouter-sandbox/Dockerfile new file mode 100644 index 00000000000..5b05cbb2e97 --- /dev/null +++ b/deploy/autorouter-sandbox/Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=ghcr.io/berriai/litellm:main-latest +FROM ${BASE_IMAGE} + +ARG SITE_PACKAGES=/app/.venv/lib/python3.13/site-packages +COPY litellm/ ${SITE_PACKAGES}/litellm/ +COPY enterprise/litellm_enterprise/ ${SITE_PACKAGES}/litellm_enterprise/ +COPY enterprise/ /app/enterprise/ +COPY litellm-proxy-extras/litellm_proxy_extras/ ${SITE_PACKAGES}/litellm_proxy_extras/ +COPY litellm-proxy-extras/ /app/litellm-proxy-extras/ +COPY schema.prisma /app/schema.prisma +COPY deploy/autorouter-sandbox/config.yaml /app/config.yaml + +CMD ["--config", "/app/config.yaml", "--port", "4000"] diff --git a/deploy/autorouter-sandbox/README.md b/deploy/autorouter-sandbox/README.md new file mode 100644 index 00000000000..6bbe9bfcae9 --- /dev/null +++ b/deploy/autorouter-sandbox/README.md @@ -0,0 +1,25 @@ +# Autorouter sandbox gateway + +A LiteLLM deployment built from this branch that you can push to and redeploy in about a minute, for iterating on the auto router without touching the real sandbox or prod + +Every model that is not defined in `config.yaml` is forwarded to the upstream gateway (`UPSTREAM_LITELLM_BASE_URL`, i.e. gateway.litellm-sandbox.ai) through `litellm_proxy/*`, so this instance inherits all upstream models and provider credentials without copying any keys. Auto routers defined here (`moe-router`) pick between those upstream models and the router code running is whatever is on this branch + +## Image + +`Dockerfile` overlays this branch's Python source onto the published `ghcr.io/berriai/litellm:main-latest` image instead of rebuilding deps and the UI, so a build takes seconds rather than ~15 minutes. If the branch adds a new dependency, switch `BASE_IMAGE` to an image built from the root `Dockerfile` + +## Run + +```bash +cp deploy/autorouter-sandbox/.env.example deploy/autorouter-sandbox/.env # fill in +docker compose -f deploy/autorouter-sandbox/docker-compose.yml up --build -d +curl localhost:4000/v1/chat/completions -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"model":"moe-router","messages":[{"role":"user","content":"hi"}]}' -i | grep x-litellm-model-name +``` + +Without Docker: `uv run --no-sync litellm --config deploy/autorouter-sandbox/config.yaml --port 4000` + +## Deploy + +`.github/workflows/deploy-autorouter-sandbox.yml` builds and pushes `ghcr.io//litellm-autorouter-sandbox:{sha,latest}` on every push to `litellm_autorouter_sandbox_deploy`. If the repo secret `AUTOROUTER_SANDBOX_KUBECONFIG_B64` is set (base64 kubeconfig; optional repo variable `AUTOROUTER_SANDBOX_NAMESPACE`), it also applies `k8s.yaml` and rolls the deployment to the new image. Create the env secret once with the command at the top of `k8s.yaml` diff --git a/deploy/autorouter-sandbox/docker-compose.yml b/deploy/autorouter-sandbox/docker-compose.yml new file mode 100644 index 00000000000..eb616d974f5 --- /dev/null +++ b/deploy/autorouter-sandbox/docker-compose.yml @@ -0,0 +1,16 @@ +services: + litellm: + image: ${LITELLM_IMAGE:-ghcr.io/berriai/litellm-autorouter-sandbox:latest} + build: + context: ../.. + dockerfile: deploy/autorouter-sandbox/Dockerfile + ports: + - "4000:4000" + env_file: + - .env + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"] + interval: 30s + timeout: 5s + retries: 3 diff --git a/deploy/autorouter-sandbox/k8s.yaml b/deploy/autorouter-sandbox/k8s.yaml new file mode 100644 index 00000000000..4601093caad --- /dev/null +++ b/deploy/autorouter-sandbox/k8s.yaml @@ -0,0 +1,60 @@ +# kubectl create secret generic litellm-autorouter-sandbox-env \ +# --from-literal=UPSTREAM_LITELLM_BASE_URL=https://gateway.litellm-sandbox.ai \ +# --from-literal=UPSTREAM_LITELLM_API_KEY=sk-... \ +# --from-literal=LITELLM_MASTER_KEY=sk-... +# kubectl apply -f k8s.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-autorouter-sandbox + labels: + app: litellm-autorouter-sandbox +spec: + replicas: 1 + selector: + matchLabels: + app: litellm-autorouter-sandbox + template: + metadata: + labels: + app: litellm-autorouter-sandbox + spec: + containers: + - name: litellm + image: ghcr.io/berriai/litellm-autorouter-sandbox:latest + imagePullPolicy: Always + ports: + - containerPort: 4000 + envFrom: + - secretRef: + name: litellm-autorouter-sandbox-env + readinessProbe: + httpGet: + path: /health/readiness + port: 4000 + initialDelaySeconds: 15 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health/liveliness + port: 4000 + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm-autorouter-sandbox +spec: + type: LoadBalancer + selector: + app: litellm-autorouter-sandbox + ports: + - port: 80 + targetPort: 4000