feat(lens): coordinate worker releases and bundled installs (#44428)

* feat(lens): coordinate worker versions and bundled installs

* test(lens): exercise bundled Compose startup and restart in CI

* fix(lens): refund failed model requests without a response

* test(lens): verify trace persistence in the bundled stack

* fix(lens): align Helm images and isolate Compose storage

* fix(lens): reject worker builds without release identity

* fix(lens): encode Compose credentials and normalize worker versions

* fix(lens): refuse worker recommendations for unidentified builds

.github/workflows/image-scan.yml

@@ -15,6 +15,9 @@ on:
       - gateway/main.py
       - backend/Dockerfile
       - backend/main.py
+      - deploy/lens/**
+      - litellm/proxy/lens/release.py
+      - tests/e2e/migrations/lens_compose_smoke.sh
       - docker/component_entrypoint.sh
       - docker/entrypoint.sh
       - litellm/proxy/prisma_migration.py
@@ -113,7 +116,7 @@ jobs:
           persist-credentials: false
 
       - name: Build runtime image
-        run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
+        run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -127,6 +130,11 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
 
+      - name: Verify the bundled Lens Compose installation and restart
+        env:
+          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
+        run: bash tests/e2e/migrations/lens_compose_smoke.sh
+
   migrations-image:
     name: migrations-image
     runs-on: ubuntu-latest

.github/workflows/lens-worker.yml

@@ -34,7 +34,14 @@ jobs:
         with:
           persist-credentials: false
       - name: Build Lens worker
-        run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+        run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+      - name: Reject custom builds without a matching release tag
+        run: |
+          if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
+            echo "::error::An unversioned worker build unexpectedly succeeded"
+            exit 1
+          fi
+          grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
       - name: Verify standalone imports with a read-only filesystem
         run: |
           docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \

Dockerfile

@@ -116,6 +116,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 
 USER root
 

backend/Dockerfile

@@ -71,6 +71,8 @@ RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component
 
 # ---------- Runtime ----------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 
 USER root
 

deploy/lens/Dockerfile

@@ -1,7 +1,10 @@
 FROM python:3.12-slim
+ARG LITELLM_RELEASE_TAG=""
+RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 WORKDIR /app
 RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
-COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/
+COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
 COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
 USER 65532:65532
 CMD ["python", "-m", "lens.worker"]

deploy/lens/README.md

@@ -2,7 +2,60 @@
 
 Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
 
-## Start a worker
+## Install the release stack
+
+Each stable, RC, and dev release containing Lens publishes the worker at the same version on GHCR and Docker Hub. Use the [LiteLLM releases page](https://github.com/BerriAI/litellm/releases) to select a version that includes the coordinated worker release
+
+For a new local installation, install Docker with Compose, download the two release files, and create a private environment file. Replace `X.Y.Z` with the release version, without `v` (RCs use `X.Y.Z-rc.N`)
+
+```bash
+mkdir litellm-lens
+cd litellm-lens
+LENS_RELEASE=X.Y.Z
+curl -fSLo compose.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/stack.yaml"
+curl -fSLo config.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/config.yaml"
+umask 077
+printf 'LITELLM_VERSION=%s\nLITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' \
+  "$LENS_RELEASE" "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+printf 'POSTGRES_PASSWORD=%s\nCLICKHOUSE_PASSWORD=%s\n' \
+  "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> .env
+docker compose up -d
+```
+
+Open `http://localhost:4000/ui/`, log in as `admin` with `LITELLM_MASTER_KEY` from `.env`, and add a model in the dashboard. In Lens, select **Connect worker**, choose that model and a monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?**, copy the worker token, and add `LENS_WORKER_TOKEN=<token>` to `.env`
+
+```bash
+docker compose --profile lens up -d
+```
+
+The stack starts LiteLLM, PostgreSQL, ClickHouse, and the worker from published images. The dashboard shows **Worker connected**. The worker has a limited token, no database credentials, and no provider keys. The stack exposes only the dashboard on localhost; use your normal ingress and managed databases for a public production deployment
+
+Keep `.env` private and preserve its salt key. Keep both named database volumes. To upgrade, wait for active investigations to finish, stop the worker, change only `LITELLM_VERSION`, then pull and recreate the stack:
+
+```bash
+docker compose --profile lens stop lens-worker
+# Update LITELLM_VERSION in .env to the new release
+docker compose --profile lens pull
+docker compose --profile lens up -d
+```
+
+This preserves your investigations, findings, model credentials, and worker token. Never use `down -v` during an upgrade. If moving from an existing installation, keep its databases and add the standalone worker instead of creating an empty replacement stack
+
+## Helm
+
+The componentized `helm/litellm` chart includes an optional Lens worker. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values:
+
+```yaml
+lensWorker:
+  enabled: true
+  tokenSecret:
+    name: litellm-lens-worker
+    key: token
+```
+
+The worker image defaults to the chart's application version, and the chart connects it to the backend service. Keep these values and the Secret when upgrading the chart so the gateway and worker upgrade together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.tag`, and `lensWorker.url`. The dashboard uses the chart's worker image for standalone install commands too
+
+## Standalone worker
 
 Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL and agent tracing. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries:
 
@@ -23,17 +76,17 @@ In **Lens > Investigations**, click **Connect worker**, choose an analysis model
 
 The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
 
-The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. CI also publishes immutable `:sha-<commit>` tags for successful worker builds on `main`. Keep the worker image compatible with your gateway version
+The dashboard selects the worker image matching the running gateway release. Release images support Linux amd64 and arm64. CI also publishes `:sha-<commit>` development images; use those only with a gateway built from the same commit and release tag
 
 After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying
 
-For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
+For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and `LITELLM_VERSION` (without `v`) in a private environment file. To use another registry, set `LENS_WORKER_IMAGE` to the compatible image instead of setting a version:
 
 ```bash
 docker compose --env-file /path/to/lens.env -f compose.yaml up -d
 ```
 
-Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`. To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000
+To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
 
 The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
 
@@ -130,3 +183,14 @@ The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`,
 Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums
 
 Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push
+
+
+## Release compatibility
+
+Released gateway and worker images carry `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release
+
+The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/litellm/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64
+
+For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version
+
+The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker

deploy/lens/compose.build.yaml

@@ -3,4 +3,6 @@ services:
     build:
       context: ../..
       dockerfile: deploy/lens/Dockerfile
+      args:
+        LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?Set the release tag used by the gateway}
     image: litellm-lens-worker:local

deploy/lens/compose.yaml

@@ -1,6 +1,6 @@
 services:
   lens-worker:
-    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b}
+    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION:?Set LITELLM_VERSION to the gateway release, without the v prefix}}
     environment:
       LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
       LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}

deploy/lens/config.yaml

@@ -0,0 +1,7 @@
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  tracing:
+    store:
+      type: clickhouse
+      url: os.environ/CLICKHOUSE_URL
+      retention_days: 14

deploy/lens/stack.yaml

@@ -0,0 +1,91 @@
+name: litellm-lens
+
+services:
+  litellm:
+    image: ghcr.io/berriai/litellm:${LITELLM_VERSION:?Set LITELLM_VERSION to a published release, without the v prefix}
+    entrypoint:
+      - python3
+      - -c
+      - |
+        import os, sys
+        from urllib.parse import quote
+        postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="")
+        clickhouse_password = quote(os.environ["CLICKHOUSE_PASSWORD"], safe="")
+        os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm"
+        os.environ["CLICKHOUSE_URL"] = f"http://default:{clickhouse_password}@clickhouse:8123"
+        os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]])
+    command: ["--config", "/app/lens-config.yaml", "--port", "4000"]
+    environment:
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password}
+      STORE_MODEL_IN_DB: "True"
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
+      LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
+    volumes:
+      - ./config.yaml:/app/lens-config.yaml:ro
+    ports:
+      - "127.0.0.1:${LITELLM_PORT:-4000}:4000"
+    networks: [proxy, storage]
+    depends_on:
+      db:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    restart: unless-stopped
+
+  lens-worker:
+    profiles: [lens]
+    image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
+    environment:
+      LITELLM_URL: http://litellm:4000
+      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
+    depends_on: [litellm]
+    networks: [proxy]
+    restart: unless-stopped
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    networks: [storage]
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    restart: unless-stopped
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:26.9.6.6
+    environment:
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    restart: unless-stopped
+    networks: [storage]
+
+networks:
+  proxy:
+  storage:
+    internal: true
+
+volumes:
+  postgres_data:
+  clickhouse_data:

docker/Dockerfile.database

@@ -113,6 +113,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 
 USER root
 

docker/Dockerfile.non_root

@@ -122,6 +122,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 WORKDIR /app
 USER root
 

helm/litellm/templates/_helpers.tpl

@@ -471,6 +471,13 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
+{{- define "litellm.lensWorker.image" -}}
+{{- $backendTag := .Values.backend.image.tag | default .Chart.AppVersion -}}
+{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
+{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
+{{- printf "%s:%s" .Values.lensWorker.image.repository $tag -}}
+{{- end -}}
+
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}

helm/litellm/templates/backend/deployment.yaml

@@ -57,6 +57,8 @@ spec:
               containerPort: 4001
               protocol: TCP
           env:
+            - name: LENS_WORKER_IMAGE
+              value: {{ include "litellm.lensWorker.image" . | quote }}
             {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
             {{- if .Values.gateway.config.create }}
             - name: CONFIG_FILE_PATH

helm/litellm/templates/lens/deployment.yaml

@@ -0,0 +1,72 @@
+{{- if .Values.lensWorker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "litellm.fullname" . }}-lens-worker
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: lens-worker
+spec:
+  replicas: {{ .Values.lensWorker.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: lens-worker
+  template:
+    metadata:
+      labels:
+        {{- include "litellm.commonLabels" . | nindent 8 }}
+        app.kubernetes.io/component: lens-worker
+    spec:
+      automountServiceAccountToken: false
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: lens-worker
+          image: {{ include "litellm.lensWorker.image" . | quote }}
+          imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: [ALL]
+          env:
+            - name: LITELLM_URL
+              value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }}
+            - name: LENS_WORKER_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ required "lensWorker.tokenSecret.name must reference a Lens worker token" .Values.lensWorker.tokenSecret.name | quote }}
+                  key: {{ .Values.lensWorker.tokenSecret.key | quote }}
+          resources:
+            {{- toYaml .Values.lensWorker.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }}
+      {{- with .Values.lensWorker.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.lensWorker.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.lensWorker.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/litellm/tests/lens_worker_tests.yaml

@@ -0,0 +1,114 @@
+suite: Lens worker release and credentials
+templates:
+  - lens/deployment.yaml
+  - backend/deployment.yaml
+  - gateway/configmap.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: keeps the worker opt in
+    template: lens/deployment.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+  - it: requires a limited worker credential when enabled
+    template: lens/deployment.yaml
+    set:
+      lensWorker.enabled: true
+    asserts:
+      - failedTemplate:
+          errorMessage: lensWorker.tokenSecret.name must reference a Lens worker token
+  - it: uses the chart release and a secret without granting Kubernetes access
+    template: lens/deployment.yaml
+    chart:
+      appVersion: v1.2.3
+    set:
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3
+      - equal:
+          path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef
+          value:
+            name: lens-credential
+            key: token
+      - equal:
+          path: spec.template.spec.automountServiceAccountToken
+          value: false
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
+          value: true
+      - equal:
+          path: spec.template.spec.volumes[0].emptyDir
+          value:
+            medium: Memory
+            sizeLimit: 1Gi
+  - it: advertises the same private dev image to standalone installers
+    template: backend/deployment.yaml
+    set:
+      lensWorker.image.repository: registry.example/lens-worker
+      lensWorker.image.tag: branch-main-1234567
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_WORKER_IMAGE
+            value: registry.example/lens-worker:branch-main-1234567
+  - it: supports an external gateway and a registry override
+    template: lens/deployment.yaml
+    set:
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+      lensWorker.url: https://gateway.example/proxy
+      lensWorker.image.repository: registry.example/lens-worker
+      lensWorker.image.tag: branch-main-1234567
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: registry.example/lens-worker:branch-main-1234567
+      - equal:
+          path: spec.template.spec.containers[0].env[0].value
+          value: https://gateway.example/proxy
+  - it: prefixes a numeric chart release with v
+    template: lens/deployment.yaml
+    chart:
+      appVersion: 1.2.3-rc.4
+    set:
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-rc.4
+  - it: follows a backend image override when no worker tag is set
+    template: lens/deployment.yaml
+    set:
+      backend.image.tag: branch-main-1234567
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:branch-main-1234567
+  - it: recommends the overridden backend release for standalone installers
+    template: backend/deployment.yaml
+    set:
+      backend.image.tag: v1.2.3-dev.4
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_WORKER_IMAGE
+            value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4
+  - it: normalizes a numeric backend tag to the published worker tag
+    template: lens/deployment.yaml
+    set:
+      backend.image.tag: 1.2.3-dev.4
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4

helm/litellm/values.yaml

@@ -629,3 +629,25 @@ ui:
   affinity: {}
   # Same shape as gateway.topologySpreadConstraints.
   topologySpreadConstraints: []
+
+lensWorker:
+  enabled: false
+  replicaCount: 1
+  image:
+    repository: ghcr.io/berriai/litellm-lens-worker
+    tag: ""
+    pullPolicy: IfNotPresent
+  tokenSecret:
+    name: ""
+    key: token
+  url: ""
+  tmpSizeLimit: 1Gi
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      memory: 2Gi
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}

litellm/proxy/_lazy_openapi_snapshot.json

@@ -34885,6 +34885,10 @@
         "WorkerCreated": {
           "additionalProperties": false,
           "properties": {
+            "image": {
+              "title": "Image",
+              "type": "string"
+            },
             "token": {
               "title": "Token",
               "type": "string"
@@ -34894,6 +34898,7 @@
             }
           },
           "required": [
+            "image",
             "worker",
             "token"
           ],

litellm/proxy/lens/endpoints.py

@@ -43,6 +43,7 @@ from litellm.proxy.lens.models import (
     Worker,
     WorkerCreated,
 )
+from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_image
 from litellm.proxy.lens.repository import LensRepository, WriterDatabase
 from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
 from litellm.proxy.lens.state import (
@@ -421,9 +422,20 @@ class WorkerName(WorkerBilling):
     name: str = Field(default="Lens worker", min_length=1)
 
 
+def configured_worker_image() -> str:
+    if image := worker_image():
+        return image
+    raise HTTPException(
+        503,
+        "This LiteLLM build has no release identity. Use a published release, make lens-dev, "
+        "or build the gateway and worker from the same commit with the same LITELLM_RELEASE_TAG.",
+    )
+
+
 @router.post("/workers/register", response_model=WorkerCreated)
 async def register_worker(body: WorkerName, auth: Auth) -> WorkerCreated:
     scope: Final = user_scope(auth, write=True)
+    image: Final = configured_worker_image()
     await validate_key(body.analysis_key_id)
     token: Final = "lens-" + secrets.token_urlsafe(40)
     worker: Final = Worker(
@@ -434,7 +446,7 @@ async def register_worker(body: WorkerName, auth: Auth) -> WorkerCreated:
         last_seen=datetime(1970, 1, 1, tzinfo=timezone.utc),
     )
     await repository().save_worker(worker, hashlib.sha256(token.encode()).hexdigest())
-    return WorkerCreated(worker=worker, token=token)
+    return WorkerCreated(worker=worker, token=token, image=image)
 
 
 @router.put("/workers/{worker_id}/billing-key", response_model=Worker)
@@ -466,9 +478,11 @@ async def revoke_worker(worker_id: str, auth: Auth) -> bool:
 
 
 @router.post("/worker/claim", response_model=Claim | None)
-async def claim(worker: WorkerAuth, protocol_version: int = 1) -> Claim | None:
-    if protocol_version not in (2, 3):
-        raise HTTPException(409, "Upgrade the Lens worker using the current Connect worker command")
+async def claim(worker: WorkerAuth, protocol_version: int = 1, worker_release: str = "") -> Claim | None:
+    image: Final = configured_worker_image()
+    expected: Final = release_tag()
+    if protocol_version != PROTOCOL_VERSION or worker_release != expected:
+        raise HTTPException(409, f"Upgrade the Lens worker to {image} and retry")
     if worker.analysis_key_id is None:
         raise HTTPException(409, "Assign an analysis key to this worker in Lens setup")
     now: Final = datetime.now(timezone.utc)

litellm/proxy/lens/models.py

@@ -252,6 +252,7 @@ class Worker(Record):
 
 
 class WorkerCreated(Record):
+    image: str
     worker: Worker
     token: str
 

litellm/proxy/lens/release.py

@@ -0,0 +1,35 @@
+import os
+from importlib.metadata import PackageNotFoundError, distribution
+from pathlib import Path
+from typing import Final
+
+PROTOCOL_VERSION: Final = 4
+
+
+def release_tag() -> str:
+    if "LITELLM_RELEASE_TAG" in os.environ:
+        return os.environ["LITELLM_RELEASE_TAG"]
+    try:
+        installed: Final = distribution("litellm")
+    except PackageNotFoundError:
+        return ""
+    if installed.read_text("direct_url.json") is not None:
+        return ""
+    if Path(str(installed.locate_file("litellm/proxy/lens/release.py"))).resolve() != Path(__file__).resolve():
+        return ""
+
+    from packaging.version import Version
+
+    parsed: Final = Version(installed.version)
+    suffix: Final = f"-dev.{parsed.dev}" if parsed.dev is not None else f"-rc.{parsed.pre[1]}" if parsed.pre else ""
+    return f"v{parsed.base_version}{suffix}"
+
+
+def worker_image() -> str:
+    tag: Final = release_tag()
+    if not tag:
+        return ""
+    override: Final = os.environ.get("LENS_WORKER_IMAGE", "")
+    if override:
+        return override
+    return f"ghcr.io/berriai/litellm-lens-worker:{tag}"

litellm/proxy/lens/worker.py

@@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError
 
 from .analysis import AnalysisResponseError, analyze_sample, validation_details
 from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult, Progress, Result, Sample
+from .release import PROTOCOL_VERSION, release_tag
 
 logger: Final = logging.getLogger("litellm.lens.worker")
 
@@ -120,8 +121,26 @@ class LensWorker:
             await self.sleep(2**attempt)
             return await self.model_request(path, body, attempt + 1)
 
+    async def report_unreadable_claim(self, identity: ClaimIdentity) -> None:
+        failure: Final = await self.client.post(
+            f"/lens/worker/{identity.lens_id}/{identity.job.id}/result",
+            json=Result(
+                coverage=Coverage(),
+                error="The worker could not read this investigation. Update the worker to match the gateway, then retry.",
+            ).model_dump(),
+        )
+        if failure.status_code != 409:
+            failure.raise_for_status()
+        logger.warning("Worker could not read a claimed investigation; reported a version compatibility failure")
+
     async def run_once(self) -> bool:
-        response: Final = await self.client.post("/lens/worker/claim", params=MappingProxyType({"protocol_version": 3}))
+        response: Final = await self.client.post(
+            "/lens/worker/claim",
+            params=MappingProxyType({"protocol_version": str(PROTOCOL_VERSION), "worker_release": release_tag()}),
+        )
+        if response.status_code == 409:
+            logger.warning("Lens worker cannot claim work: %s", response.text)
+            return False
         response.raise_for_status()
         payload: Final = response.json()
         if payload is None:
@@ -129,17 +148,7 @@ class LensWorker:
         try:
             claim: Final = Claim.model_validate(payload)
         except ValidationError:
-            identity: Final = ClaimIdentity.model_validate(payload)
-            failure: Final = await self.client.post(
-                f"/lens/worker/{identity.lens_id}/{identity.job.id}/result",
-                json=Result(
-                    coverage=Coverage(),
-                    error="The worker could not read this investigation. Update the worker to match the gateway, then retry.",
-                ).model_dump(),
-            )
-            if failure.status_code != 409:
-                failure.raise_for_status()
-            logger.warning("Worker could not read a claimed investigation; reported a version compatibility failure")
+            await self.report_unreadable_claim(ClaimIdentity.model_validate(payload))
             return True
         prefix: Final = f"/lens/worker/{claim.lens_id}/{claim.job.id}"
 

scripts/lens_dev.sh

@@ -13,6 +13,7 @@
 set -euo pipefail
 
 repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+source_release_tag="sha-$(git -C "$repo_root" rev-parse HEAD)"
 proxy_port="${LENS_DEV_PROXY_PORT:-4000}"
 ui_port="${LENS_DEV_UI_PORT:-3000}"
 state_dir="${LENS_DEV_STATE_DIR:-$repo_root/.lens-dev}"
@@ -108,6 +109,8 @@ proxy_env() {
   unset ANTHROPIC_BASE_URL ANTHROPIC_AUTH_TOKEN ANTHROPIC_CUSTOM_HEADERS OPENAI_BASE_URL OPENAI_API_BASE
   for var in $(compgen -e | grep '^REDIS_' || true); do unset "$var"; done
   eval "$1"
+  export LITELLM_RELEASE_TAG="$source_release_tag"
+  export LENS_WORKER_IMAGE=litellm-lens-worker:local
   export LITELLM_MODE=PRODUCTION
   export LITELLM_MASTER_KEY="$master_key"
   if [ "$master_key" = sk-1234 ]; then export LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true; fi
@@ -236,7 +239,8 @@ main() {
   wait_for_proxy "$proxy_pid"
   ensure_worker_token
 
-  LITELLM_MODE=PRODUCTION LITELLM_URL="$proxy_url" LENS_WORKER_TOKEN="$(cat "$token_file")" \
+  LITELLM_RELEASE_TAG="$source_release_tag" \
+    LITELLM_MODE=PRODUCTION LITELLM_URL="$proxy_url" LENS_WORKER_TOKEN="$(cat "$token_file")" \
     "$py" -c "import asyncio, logging; from litellm.proxy.lens.worker import main; logging.basicConfig(level=logging.INFO); asyncio.run(main())" \
     < /dev/null > "$log_dir/worker.log" 2>&1 &
   pids+=("$!")

tests/e2e/migrations/lens_compose_smoke.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+qa_dir=$(mktemp -d)
+master_key="sk-$(openssl rand -hex 32)"
+compose=(docker compose -p lens-compose-ci --env-file "$qa_dir/env" -f deploy/lens/stack.yaml)
+cleanup() {
+  "${compose[@]}" --profile lens down -v --remove-orphans >/dev/null 2>&1 || true
+  rm -rf "$qa_dir"
+}
+trap cleanup EXIT
+umask 077
+printf 'LITELLM_VERSION=0.0.0-lens-ci\nLITELLM_PORT=4418\nLITELLM_MASTER_KEY=%s\nLITELLM_SALT_KEY=sk-%s\n' \
+  "$master_key" "$(openssl rand -hex 32)" > "$qa_dir/env"
+printf 'POSTGRES_PASSWORD=%s:/?#@%%\nCLICKHOUSE_PASSWORD=%s:/?#@%%\n' \
+  "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> "$qa_dir/env"
+docker tag "${LITELLM_IMAGE:?Set LITELLM_IMAGE to the built gateway image}" ghcr.io/berriai/litellm:0.0.0-lens-ci
+docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f deploy/lens/Dockerfile \
+  -t ghcr.io/berriai/litellm-lens-worker:v0.0.0-lens-ci .
+"${compose[@]}" up -d
+
+api() {
+  curl --fail-with-body --silent --show-error --max-time 30 \
+    -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \
+    "http://127.0.0.1:4418$1" "${@:2}"
+}
+ready=false
+for attempt in $(seq 1 90); do
+  if api /health/liveliness > /dev/null 2>&1; then ready=true; break; fi
+  sleep 2
+done
+if [[ "$ready" != true ]]; then "${compose[@]}" logs litellm; exit 1; fi
+trace_id=$(openssl rand -hex 16)
+span_id=$(openssl rand -hex 8)
+start_ns="$(date +%s)000000000"
+jq -n --arg trace "$trace_id" --arg span "$span_id" --arg at "$start_ns" \
+  '{resourceSpans:[{resource:{attributes:[{key:"service.name",value:{stringValue:"lens-compose-ci"}}]},
+    scopeSpans:[{scope:{name:"lens-compose-ci"},spans:[{traceId:$trace,spanId:$span,name:"Compose trace",
+      kind:1,startTimeUnixNano:$at,endTimeUnixNano:$at,
+      attributes:[{key:"openinference.span.kind",value:{stringValue:"AGENT"}}],status:{code:1}}]}]}]}' \
+  > "$qa_dir/trace.json"
+api /v1/traces -d "@$qa_dir/trace.json" > /dev/null
+trace_saved() {
+  for attempt in $(seq 1 60); do
+    if api "/v1/traces/$trace_id" > "$qa_dir/saved-trace.json" 2>/dev/null && \
+      jq -e --arg trace "$trace_id" --arg span "$span_id" \
+        '.summary.trace_id == $trace and any(.spans[]; .span_id == $span)' "$qa_dir/saved-trace.json" > /dev/null; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+trace_saved
+api /key/generate -d '{"key_alias":"Lens Compose CI","models":["lens-compose-ci"],"max_budget":1}' > "$qa_dir/key.json"
+key_id=$(jq -r '.token_id // empty' "$qa_dir/key.json")
+if [[ -z "$key_id" ]]; then
+  key_id=$(jq -rj '.key' "$qa_dir/key.json" | openssl dgst -sha256 | awk '{print $NF}')
+fi
+jq -n --arg key "$key_id" '{name:"Lens Compose CI",analysis_key_id:$key}' > "$qa_dir/registration.json"
+api /lens/workers/register -d "@$qa_dir/registration.json" > "$qa_dir/worker.json"
+jq -e '.image == "ghcr.io/berriai/litellm-lens-worker:v0.0.0-lens-ci"' "$qa_dir/worker.json" > /dev/null
+printf 'LENS_WORKER_TOKEN=%s\n' "$(jq -r '.token' "$qa_dir/worker.json")" >> "$qa_dir/env"
+worker_id=$(jq -r '.worker.id' "$qa_dir/worker.json")
+heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S')
+"${compose[@]}" --profile lens up -d
+
+connected() {
+  for attempt in $(seq 1 60); do
+    if api /lens > "$qa_dir/lens.json" 2>/dev/null && \
+      jq -e --arg id "$worker_id" --arg since "$heartbeat_after" \
+        '.workers[] | select(.id == $id and .last_seen > $since)' "$qa_dir/lens.json" > /dev/null; then
+      return 0
+    fi
+    sleep 2
+  done
+  "${compose[@]}" --profile lens logs lens-worker
+  return 1
+}
+connected
+printf 'Fresh Compose stack: matching worker image and authenticated heartbeat passed\n'
+
+for target in db:5432 clickhouse:8123; do
+  service=${target%:*}
+  port=${target#*:}
+  address=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$("${compose[@]}" ps -q "$service")")
+  "${compose[@]}" exec -T lens-worker python -c '
+import socket, sys
+for host in (sys.argv[1], sys.argv[2]):
+    try:
+        connection = socket.create_connection((host, int(sys.argv[3])), timeout=2)
+    except OSError:
+        continue
+    connection.close()
+    raise SystemExit("Worker can reach a datastore directly")
+' "$service" "$address" "$port"
+done
+printf 'Worker can reach the proxy but cannot connect directly to PostgreSQL or ClickHouse\n'
+
+status=$(curl --silent --show-error -o "$qa_dir/mismatch.json" -w '%{http_code}' -X POST \
+  -H "Authorization: Bearer $(jq -r '.token' "$qa_dir/worker.json")" \
+  'http://127.0.0.1:4418/lens/worker/claim?protocol_version=4&worker_release=v0.0.0-old')
+[[ "$status" == 409 ]]
+jq -e '.detail | contains("Upgrade the Lens worker")' "$qa_dir/mismatch.json" > /dev/null
+
+"${compose[@]}" --profile lens restart litellm lens-worker
+heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S')
+connected
+trace_saved
+api /lens > "$qa_dir/restarted.json"
+jq -e --arg id "$worker_id" --arg key "$key_id" \
+  '.workers[] | select(.id == $id and .analysis_key_id == $key)' "$qa_dir/restarted.json" > /dev/null
+printf 'Compose restart: trace, worker identity, token and billing assignment preserved; wrong release rejected\n'
+
+cat > "$qa_dir/unversioned.yaml" <<'EOF'
+services:
+  litellm:
+    environment:
+      LITELLM_RELEASE_TAG: ""
+EOF
+"${compose[@]}" -f "$qa_dir/unversioned.yaml" up -d litellm
+for attempt in $(seq 1 90); do
+  if api /health/liveliness > /dev/null 2>&1; then break; fi
+  sleep 2
+done
+api /health/liveliness > /dev/null
+status=$(curl --silent --show-error --max-time 30 -o "$qa_dir/unversioned-registration.json" -w '%{http_code}' \
+  -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \
+  -d "@$qa_dir/registration.json" 'http://127.0.0.1:4418/lens/workers/register')
+[[ "$status" == 503 ]]
+jq -e '.detail | contains("no release identity")' "$qa_dir/unversioned-registration.json" > /dev/null
+status=$(curl --silent --show-error --max-time 30 -o "$qa_dir/unversioned-claim.json" -w '%{http_code}' -X POST \
+  -H "Authorization: Bearer $(jq -r '.token' "$qa_dir/worker.json")" \
+  'http://127.0.0.1:4418/lens/worker/claim?protocol_version=4&worker_release=')
+[[ "$status" == 503 ]]
+jq -e '.detail | contains("no release identity")' "$qa_dir/unversioned-claim.json" > /dev/null
+api /lens > "$qa_dir/unversioned-workers.json"
+jq -e --arg id "$worker_id" '.workers | length == 1 and .[0].id == $id' "$qa_dir/unversioned-workers.json" > /dev/null
+"${compose[@]}" up -d litellm
+heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S')
+connected
+trace_saved
+printf 'Unversioned gateway: setup and claims refused without guessing; original worker and trace recovered\n'

tests/proxy_behavior/lens/test_lifecycle.py

@@ -29,13 +29,15 @@ from litellm.proxy.lens.models import (
     Scope,
     Worker,
 )
+from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag
 from litellm.proxy.lens.repository import Database, LensRepository, Row
 from litellm.proxy.lens.state import can_access
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 
 
 @pytest_asyncio.fixture(loop_scope="function")
-async def lens_database() -> AsyncIterator[PrismaClient]:
+async def lens_database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[PrismaClient]:
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", "v0.0.0-lens-lifecycle")
     original_db: Final = proxy_server.prisma_client
     original_router: Final = proxy_server.llm_router
     original_settings: Final = proxy_server.general_settings
@@ -349,8 +351,9 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database:
         authenticated_legacy: Final = await endpoints.worker_auth(credentials)
         assert authenticated_legacy.analysis_key_id is None
         with pytest.raises(HTTPException) as needs_billing:
-            await endpoints.claim(authenticated_legacy, protocol_version=2)
+            await endpoints.claim(authenticated_legacy, protocol_version=PROTOCOL_VERSION, worker_release=release_tag())
         assert needs_billing.value.status_code == 409
+        assert "Assign an analysis key" in needs_billing.value.detail
         assert await endpoints.heartbeat(lens.id, claimed.job.id, authenticated_legacy)
         finished: Final = await endpoints.result(
             lens.id, claimed.job.id, Result(coverage=Coverage(screened=2)), authenticated_legacy, storage=None
@@ -441,6 +444,7 @@ async def test_failed_model_requests_release_lens_budget_reservations(lens_datab
         stored: Final = await endpoints.get_lens(lens.id, worker.scope)
         assert stored.spent == 0
         assert stored.jobs[0].cost == 0
+        assert not any(step.kind == "model" for step in stored.jobs[0].steps)
     finally:
         await lens_database.db.execute_raw('DELETE FROM "LiteLLM_LensRun" WHERE lens_id=$1', lens.id)
         await lens_database.db.execute_raw('DELETE FROM "LiteLLM_Lens" WHERE id=$1', lens.id)

tests/unit/proxy/lens/test_endpoints.py

@@ -6,16 +6,16 @@ from fastapi import HTTPException
 from pydantic import ValidationError
 
 import litellm
-from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm import Router
+from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.lens.endpoints import (
     list_agents,
     run_settings,
     run_window,
     user_scope,
+    validate_model,
     watchable,
     watching,
-    validate_model,
     worker_supports_model,
 )
 from litellm.proxy.lens.models import Lens, LensSettings, RunRequest, Scope
@@ -159,13 +159,17 @@ def test_invalid_explicit_execution_ids_are_rejected(identity: str) -> None:
     assert error.value.status_code == 422
 
 
+@pytest.mark.parametrize("protocol_version", (1, 2, 3))
 @pytest.mark.asyncio
-async def test_incompatible_worker_is_rejected_before_claiming_work() -> None:
+async def test_incompatible_worker_is_rejected_before_claiming_work(
+    protocol_version: int, monkeypatch: pytest.MonkeyPatch
+) -> None:
     from litellm.proxy.lens.endpoints import claim
     from tests.unit.proxy.lens.test_state import worker
 
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3")
     with pytest.raises(HTTPException) as error:
-        await claim(worker(), protocol_version=1)
+        await claim(worker(), protocol_version=protocol_version)
     assert error.value.status_code == 409
     assert "Upgrade" in error.value.detail
 
@@ -291,3 +295,38 @@ async def test_preview_reports_calendar_overflow_as_a_validation_error() -> None
         await preview_sample(body, UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), None)
     assert error.value.status_code == 422
     assert "supported calendar range" in error.value.detail
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("worker_release", ("", "v1.2.2", "branch-main-old"))
+async def test_different_release_is_rejected_before_accessing_jobs(
+    monkeypatch: pytest.MonkeyPatch, worker_release: str
+) -> None:
+    from litellm.proxy.lens.endpoints import claim
+    from litellm.proxy.lens.release import PROTOCOL_VERSION
+    from tests.unit.proxy.lens.test_state import worker
+
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3")
+    monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False)
+    with pytest.raises(HTTPException) as error:
+        await claim(worker(), protocol_version=PROTOCOL_VERSION, worker_release=worker_release)
+    assert error.value.status_code == 409
+    assert "ghcr.io/berriai/litellm-lens-worker:v1.2.3" in error.value.detail
+
+
+@pytest.mark.asyncio
+async def test_unknown_gateway_release_refuses_registration_and_claims(monkeypatch: pytest.MonkeyPatch) -> None:
+    from litellm.proxy.lens.endpoints import WorkerName, claim, register_worker
+    from litellm.proxy.lens.release import PROTOCOL_VERSION
+    from tests.unit.proxy.lens.test_state import worker
+
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", "")
+    monkeypatch.setenv("LENS_WORKER_IMAGE", "registry.example/lens-worker:old")
+    with pytest.raises(HTTPException) as registration_error:
+        await register_worker(WorkerName(analysis_key_id="a" * 64), UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN))
+    assert registration_error.value.status_code == 503
+    assert "LITELLM_RELEASE_TAG" in registration_error.value.detail
+    with pytest.raises(HTTPException) as claim_error:
+        await claim(worker(), protocol_version=PROTOCOL_VERSION, worker_release="")
+    assert claim_error.value.status_code == 503
+    assert claim_error.value.detail == registration_error.value.detail

tests/unit/proxy/lens/test_release.py

@@ -0,0 +1,74 @@
+from importlib.metadata import Distribution, PackageNotFoundError, PathDistribution
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+from litellm.proxy.lens.release import worker_image
+
+
+@pytest.mark.parametrize("tag", ("v1.2.3", "v1.2.3-rc.4", "v1.2.3-dev.5", "branch-main-1234567"))
+def test_install_command_follows_the_gateway_release(monkeypatch: pytest.MonkeyPatch, tag: str) -> None:
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", tag)
+    monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False)
+    assert worker_image() == f"ghcr.io/berriai/litellm-lens-worker:{tag}"
+
+
+def test_private_registry_override_keeps_its_exact_digest(monkeypatch: pytest.MonkeyPatch) -> None:
+    image: Final = "registry.example/lens-worker@sha256:" + "a" * 64
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", "branch-main-1234567")
+    monkeypatch.setenv("LENS_WORKER_IMAGE", image)
+    assert worker_image() == image
+
+
+@pytest.mark.parametrize(
+    "installed,expected",
+    (("1.2.3", "v1.2.3"), ("1.2.3rc4", "v1.2.3-rc.4"), ("1.2.3.dev5", "v1.2.3-dev.5")),
+)
+def test_python_installs_recommend_the_matching_worker(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, installed: str, expected: str
+) -> None:
+    from litellm.proxy.lens import release
+
+    metadata: Final = tmp_path / "litellm.dist-info"
+    metadata.mkdir()
+    metadata.joinpath("METADATA").write_text(f"Name: litellm\nVersion: {installed}\n")
+
+    def installed_distribution(name: str) -> Distribution:
+        assert name == "litellm"
+        return PathDistribution(metadata)
+
+    monkeypatch.delenv("LITELLM_RELEASE_TAG", raising=False)
+    monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False)
+    monkeypatch.setattr(release, "distribution", installed_distribution)
+    monkeypatch.setattr(release, "__file__", str(tmp_path / "litellm/proxy/lens/release.py"))
+    assert release.release_tag() == expected
+    assert worker_image() == f"ghcr.io/berriai/litellm-lens-worker:{expected}"
+
+
+@pytest.mark.parametrize("source", ("checkout", "direct-install", "unversioned-container", "missing-package"))
+def test_unknown_source_never_falls_back_to_a_package_version_or_image_override(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, source: str
+) -> None:
+    from litellm.proxy.lens import release
+
+    metadata: Final = tmp_path / "litellm.dist-info"
+    metadata.mkdir()
+    metadata.joinpath("METADATA").write_text("Name: litellm\nVersion: 1.2.3\n")
+    if source == "direct-install":
+        metadata.joinpath("direct_url.json").write_text('{"url":"file:///checkout","dir_info":{"editable":true}}')
+
+    def installed_distribution(name: str) -> Distribution:
+        if source == "missing-package":
+            raise PackageNotFoundError(name)
+        return PathDistribution(metadata)
+
+    monkeypatch.delenv("LITELLM_RELEASE_TAG", raising=False)
+    monkeypatch.setenv("LENS_WORKER_IMAGE", "registry.example/lens-worker:old")
+    monkeypatch.setattr(release, "distribution", installed_distribution)
+    if source != "checkout":
+        monkeypatch.setattr(release, "__file__", str(tmp_path / "litellm/proxy/lens/release.py"))
+    if source == "unversioned-container":
+        monkeypatch.setenv("LITELLM_RELEASE_TAG", "")
+    assert release.release_tag() == ""
+    assert worker_image() == ""

tests/unit/proxy/lens/test_worker.py

@@ -417,3 +417,22 @@ async def test_transient_heartbeat_failure_recovers_without_cancelling_analysis(
     assert result.error == ""
     assert result.coverage.screened == 1 and result.coverage.unassessable == 0
     assert attempts.qsize() == 2 and saved.empty()
+
+
+@pytest.mark.asyncio
+async def test_worker_announces_release_and_waits_on_incompatible_gateway(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    from litellm.proxy.lens.release import PROTOCOL_VERSION
+
+    monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3")
+
+    def handle(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/lens/worker/claim"
+        assert request.url.params["protocol_version"] == str(PROTOCOL_VERSION)
+        assert request.url.params["worker_release"] == "v1.2.3"
+        return httpx.Response(409, json={"detail": "Upgrade the Lens worker to v1.2.4"})
+
+    async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client:
+        assert not await LensWorker(client).run_once()
+    assert "Upgrade the Lens worker to v1.2.4" in caplog.text

tests/unit/test_lens_dev.py

@@ -2,6 +2,7 @@ import os
 import subprocess
 import sys
 from pathlib import Path
+from typing import Final
 
 ROOT = Path(__file__).resolve().parents[2]
 SCRIPT = ROOT / "scripts" / "lens_dev.sh"
@@ -123,6 +124,19 @@ def test_proxy_env_permits_the_weak_key_only_when_chosen(tmp_path):
     assert "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true" in proc.stdout
 
 
+def test_source_development_overrides_an_inherited_release_with_its_own_commit(tmp_path: Path) -> None:
+    proc: Final = _run(
+        tmp_path,
+        'proxy_env "export LITELLM_RELEASE_TAG=v0.0.0-old"; '
+        'test "$LITELLM_RELEASE_TAG" = "sha-$(git -C "$repo_root" rev-parse HEAD)"; '
+        'printf "%s" "$LENS_WORKER_IMAGE"',
+        LITELLM_RELEASE_TAG="v0.0.0-old",
+        LENS_WORKER_IMAGE="registry.example/lens-worker:old",
+    )
+    assert proc.returncode == 0, proc.stderr
+    assert proc.stdout == "litellm-lens-worker:local"
+
+
 def test_external_database_url_never_starts_compose_postgres(tmp_path):
     docker = tmp_path / "bin" / "docker"
     proc = _run(

ui/litellm-dashboard/src/components/lens/setup/worker/WorkerDialog.integration.test.tsx

@@ -12,6 +12,7 @@ vi.mock("@/components/networking", () => ({
 
 const created = {
   token: "lens-test-token",
+  image: "ghcr.io/berriai/litellm-lens-worker:v1.2.3",
   worker: {
     id: "worker",
     name: "Lens worker",
@@ -57,7 +58,11 @@ describe("Worker setup", () => {
     expect(command).toContain("LITELLM_URL=https://gateway.example/proxy");
     expect(command).toContain("LENS_WORKER_TOKEN=lens-test-token");
     expect(command).toContain("--add-host host.docker.internal:host-gateway");
-    expect(command).toContain("ghcr.io/berriai/litellm-lens-worker@sha256:");
+    expect(command).toContain(created.image);
+    await user.click(screen.getByText("Using Docker Compose or Helm?"));
+    await user.click(screen.getByRole("button", { name: "Copy worker token" }));
+    expect(await navigator.clipboard.readText()).toBe(created.token);
+    expect(screen.getByRole("button", { name: "Token copied" })).toBeVisible();
   });
   it("assigns billing to an existing worker without replacing its access token", async () => {
     const user = userEvent.setup();

ui/litellm-dashboard/src/components/lens/setup/worker/WorkerInstall.tsx

@@ -1,5 +1,6 @@
 "use client";
 
+import { useState } from "react";
 import { Button } from "@/components/ui/button";
 
 import { CheckCircle2, Copy, Loader2 } from "lucide-react";
@@ -25,6 +26,7 @@ export function WorkerInstall({
   onReady?: () => void;
   onClose: () => void;
 }) {
+  const [tokenCopied, setTokenCopied] = useState(false);
   return (
     <div className="min-w-0 space-y-5">
       {!connected && (
@@ -34,7 +36,7 @@ export function WorkerInstall({
             className="w-full gap-2"
             onClick={async () => {
               try {
-                await navigator.clipboard.writeText(workerSetupCommand(address, created.token));
+                await navigator.clipboard.writeText(workerSetupCommand(address, created.token, created.image));
                 setCopied(true);
               } catch {
                 setError("Clipboard access failed. Allow clipboard access and try again.");
@@ -51,9 +53,30 @@ export function WorkerInstall({
               aria-label="Docker command preview"
               className="mt-3 max-h-48 overflow-auto rounded-md bg-muted/40 p-3 text-xs leading-5"
             >
-              {workerSetupCommand(address, created.token)}
+              {workerSetupCommand(address, created.token, created.image)}
             </pre>
           </details>
+          <details className="text-sm">
+            <summary className="cursor-pointer text-muted-foreground">Using Docker Compose or Helm?</summary>
+            <p className="mt-3 text-muted-foreground">
+              Save this private token as LENS_WORKER_TOKEN in Compose or in your Helm worker token secret. Keep it for
+              future upgrades.
+            </p>
+            <Button
+              variant="outline"
+              className="mt-3"
+              onClick={async () => {
+                try {
+                  await navigator.clipboard.writeText(created.token);
+                  setTokenCopied(true);
+                } catch {
+                  setError("Clipboard access failed. Allow clipboard access and try again.");
+                }
+              }}
+            >
+              {tokenCopied ? "Token copied" : "Copy worker token"}
+            </Button>
+          </details>
         </>
       )}
       {!connected && (

ui/litellm-dashboard/src/components/lens/setup/worker/workerCommand.ts

@@ -1,23 +1,20 @@
 import { proxyBaseUrl } from "@/components/networking";
 import { serverRootPath } from "@/lib/serverRootPath";
 
-export const LENS_WORKER_IMAGE =
-  "ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b";
-
 export function initialProxyAddress(): string {
   const url = new URL(proxyBaseUrl || serverRootPath, window.location.origin);
   if (["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)) url.hostname = "host.docker.internal";
   return url.toString().replace(/\/$/, "");
 }
 
-export function workerSetupCommand(address: string, token: string): string {
+export function workerSetupCommand(address: string, token: string, image: string): string {
   const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'";
   return [
     "docker run -d --restart unless-stopped --read-only --cap-drop ALL",
     "  --tmpfs /tmp:rw,noexec,nosuid,size=1g",
-    "  --security-opt no-new-privileges --platform linux/amd64 --add-host host.docker.internal:host-gateway",
+    "  --security-opt no-new-privileges --add-host host.docker.internal:host-gateway",
     `  -e ${quote("LITELLM_URL=" + address)}`,
     `  -e ${quote("LENS_WORKER_TOKEN=" + token)}`,
-    `  ${LENS_WORKER_IMAGE}`,
+    `  ${quote(image)}`,
   ].join(" \\\n");
 }

ui/litellm-dashboard/src/components/lens/setup/worker/workerSchema.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { validateWorkerAddress, analysisAccessSchema, workerFormSchema } from "./workerSchema";
-import { workerSetupCommand, LENS_WORKER_IMAGE } from "./workerCommand";
+import { workerSetupCommand } from "./workerCommand";
 
 const workerDefaults = {
   useExisting: false,
@@ -26,14 +26,13 @@ describe("worker setup", () => {
   });
 
   it("quotes apostrophes literally and retains the pinned image and runtime restrictions", () => {
-    const command = workerSetupCommand("https://gateway.example/proxy?name=it's", "token'quoted");
+    const image = "registry.example/lens-worker:v1.2.3-rc.4";
+    const command = workerSetupCommand("https://gateway.example/proxy?name=it's", "token'quoted", image);
     expect(command).toContain("'LITELLM_URL=https://gateway.example/proxy?name=it'\\''s'");
     expect(command).toContain("'LENS_WORKER_TOKEN=token'\\''quoted'");
     expect(command).toContain("--read-only --cap-drop ALL");
-    expect(command).toContain(
-      "--security-opt no-new-privileges --platform linux/amd64 --add-host host.docker.internal:host-gateway",
-    );
-    expect(command.split("\n").at(-1)?.trim()).toBe(LENS_WORKER_IMAGE);
+    expect(command).toContain("--security-opt no-new-privileges --add-host host.docker.internal:host-gateway");
+    expect(command.split("\n").at(-1)?.trim()).toBe(`'${image}'`);
   });
 });
 

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -50091,6 +50091,8 @@ export interface components {
         };
         /** WorkerCreated */
         WorkerCreated: {
+            /** Image */
+            image: string;
             /** Token */
             token: string;
             worker: components["schemas"]["Worker"];
@@ -62899,6 +62901,7 @@ export interface operations {
         parameters: {
             query?: {
                 protocol_version?: number;
+                worker_release?: string;
             };
             header?: never;
             path?: never;