refactor(helm)!: connect both charts to a separately deployed Lens (#49169)

Neither chart bundles Lens anymore. The Lens subchart, its lock entry,
the vendored chart archive, the templates/lens resources and the
bundled-Lens install smoke are gone. A connection-only lens block wires
the gateway and backend to a Lens deployed from its own chart, and an
enabled lensWorker value fails the render instead of being ignored.

Supersedes #47869, which made the same change for helm/litellm only.
This commit is contained in:
yuneng-jiang 2026-10-29 19:29:44 -07:00 • committed by GitHub
parent 44cd858697
commit 85af2f65c9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 356 additions and 2899 deletions

View file

@ -1,152 +0,0 @@
name: Lens installation smoke
on:
workflow_dispatch:
inputs:
lens_image:
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
required: true
type: string
lens_upgrade_image:
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
required: true
type: string
gateway_baseline_ref:
description: Previously qualified gateway source commit, full SHA
required: true
type: string
permissions:
contents: read
concurrency:
group: lens-install-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-images:
runs-on: ubuntu-latest-16-cores
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include:
- component: gateway
dockerfile: gateway/Dockerfile
- component: backend
dockerfile: backend/Dockerfile
- component: ui
dockerfile: ui/Dockerfile
- component: migrations
dockerfile: migrations/Dockerfile
- component: monolith
dockerfile: Dockerfile
- component: gateway
dockerfile: gateway/Dockerfile
baseline: true
- component: backend
dockerfile: backend/Dockerfile
baseline: true
- component: monolith
dockerfile: Dockerfile
baseline: true
env:
COMPONENT: ${{ matrix.component }}
DOCKERFILE: ${{ matrix.dockerfile }}
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
CURRENT_REF: ${{ github.sha }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ env.SOURCE_REF }}
- name: Verify the selected source
shell: bash -euo pipefail {0}
run: |
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
test "$SOURCE_REF" != "$CURRENT_REF"
fi
- name: Build the matching release image
run: |
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
- name: Save the matching release image
run: |
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
compression-level: 0
retention-days: 3
if-no-files-found: error
overwrite: true
helm-install:
needs: build-images
runs-on: ubuntu-latest-16-cores
timeout-minutes: 25
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: lens-install-*-${{ github.sha }}
merge-multiple: true
path: ${{ runner.temp }}/lens-install-images
- name: Load the matching release images
run: |
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
gzip -dc "$archive" | docker load
rm "$archive"
done
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
with:
cosign-release: v2.4.3
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ vars.LENS_READ_USER || github.actor }}
password: ${{ secrets.LENS_READ_TOKEN }}
- name: Verify independent Lens images before loading them
env:
LENS_IMAGE: ${{ inputs.lens_image }}
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
shell: bash -euo pipefail {0}
run: |
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
docker pull "$image"
done
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
- name: Install pinned Kubernetes test tools
run: |
curl --fail --location --output "$RUNNER_TEMP/kind" \
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
chmod +x "$RUNNER_TEMP/kind"
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
chmod +x "$RUNNER_TEMP/kubectl"
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
- name: Install, ingest, upgrade, and restart both charts
run: bash tests/e2e/migrations/lens_helm_smoke.sh

View file

@ -5,8 +5,5 @@ dependencies:
- name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: 18.19.1
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b
generated: '2026-10-09T04:18:10.643228000Z'
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
generated: '2026-10-30T02:13:11.505726000Z'

View file

@ -39,6 +39,3 @@ dependencies:
version: "18.19.1"
repository: oci://registry-1.docker.io/bitnamicharts
condition: redis.enabled
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

View file

@ -256,9 +256,7 @@ itself.
## Connect Lens
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
## Shutdown accounting

View file

@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -402,95 +398,28 @@ shutdown drain window.
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.labels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}}
{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

View file

@ -1,8 +1,6 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "litellm.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- $lensMode := include "litellm.lens.mode" . -}}
{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
@ -46,18 +44,18 @@ spec:
- host: {{ .host | quote }}
http:
paths:
{{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }}
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
name: {{ $.Values.lens.service.name }}
port:
number: {{ $.Values.lensWorker.service.port }}
number: {{ $.Values.lens.service.port }}
{{- else }}
serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }}
servicePort: {{ $.Values.lensWorker.service.port }}
serviceName: {{ $.Values.lens.service.name }}
servicePort: {{ $.Values.lens.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

View file

@ -0,0 +1,128 @@
suite: lens connection
templates:
- deployment.yaml
- configmap-litellm.yaml
- ingress.yaml
tests:
- it: connects the proxy to an external Lens
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service ahead of the host paths
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- equal:
path: spec.rules[0].http.paths[0].path
value: /lens-ingest
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

View file

@ -1,84 +0,0 @@
suite: Lens endpoint defaults
templates:
- deployment.yaml
- configmap-litellm.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest

View file

@ -1,187 +0,0 @@
---
suite: Lens deployment modes
templates:
- ingress.yaml
- configmap-litellm.yaml
- deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}]
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

View file

@ -1,52 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- configmap-litellm.yaml
- deployment.yaml
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

View file

@ -1,59 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=

View file

@ -1,167 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- configmap-litellm.yaml
- deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects deployment.yaml to the shared Lens service
template: deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.hosts:
- host: gateway.example
paths:
- path: /
pathType: Prefix
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in deployment.yaml
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference

View file

@ -1,162 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60

View file

@ -668,70 +668,15 @@ serviceMonitor:
# - test-namespace
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -1,6 +0,0 @@
dependencies:
- name: lens
repository: oci://ghcr.io/berriai/charts
version: 0.1.0-dev.0
digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf
generated: '2026-10-09T04:18:10.642706000Z'

View file

@ -6,8 +6,3 @@ version: 0.1.0
appVersion: "0.1.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
dependencies:
- name: lens
version: 0.1.0-dev.0
repository: oci://ghcr.io/berriai/charts

View file

@ -1,10 +1,8 @@
# LiteLLM componentized chart
This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service
This chart deploys separate gateway, backend and UI services
Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured.

View file

@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and
collector containers through an emptyDir. Empty when the sidecar is off
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
*/}}
{{- define "litellm.lensWorker.image" -}}
{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
@ -635,93 +631,27 @@ shutdown drain window.
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- $mode := include "litellm.lens.mode" . -}}
{{- if ne $mode "disabled" }}
{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}}
{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}}
{{- fail "lensWorker.gateway.secretName is required for external Lens" -}}
{{- end -}}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }}
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ include "litellm.lensWorker.publicUrl" . | quote }}
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }}
key: {{ .Values.lensWorker.gateway.secretKey | quote }}
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }}
key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }}
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.lensWorker.labels" -}}
{{- $labels := include "litellm.commonLabels" . | fromYaml -}}
{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}}
{{- toYaml $labels -}}
{{- end -}}
{{- define "litellm.lensWorker.serviceTokenSecretName" -}}
{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}}
{{- end -}}
{{- define "litellm.lensWorker.gatewaySecretName" -}}
{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}}
{{- end -}}
{{- define "litellm.lensWorker.bundledClickhouse" -}}
{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.publicUrl" -}}
{{- if .Values.lensWorker.publicUrl -}}
{{- .Values.lensWorker.publicUrl -}}
{{- else if eq (include "litellm.lens.mode" .) "external" -}}
{{- fail "lensWorker.publicUrl is required for external Lens" -}}
{{- else if .Values.lensWorker.ingress.enabled -}}
{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}}
{{- else if and .Values.ingress.enabled .Values.ingress.host -}}
{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}}
{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}}
{{- else -}}
{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.lensWorker.clickhouseName" -}}
{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}}
{{- end -}}
{{- define "litellm.lens.mode" -}}
{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}}
{{- if not (has $mode (list "bundled" "external" "disabled")) -}}
{{- fail "lensWorker.mode must be bundled, external or disabled" -}}
{{- end -}}
{{- $mode -}}
{{- end -}}
{{- define "litellm.lens.render" -}}
{{- $root := .root -}}
{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}}
{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}}
{{- $_ := set $values "component" "lens-worker" -}}
{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}}
{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}}
{{- $_ := set $values.gateway "enabled" true -}}
{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}}
{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}}
{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}}
{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}}
{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}}
{{- $_ := set $values "ingestionUrl" $ingestion -}}
{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}}
{{- end -}}
{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}}
{{- end -}}

View file

@ -166,14 +166,14 @@ spec:
- host: {{ $componentHosts.gateway | quote }}
http:
paths:
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
- path: /
pathType: Prefix
@ -249,15 +249,15 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }}
{{- if and .Values.lens.enabled .Values.lens.service.name }}
{{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }}
- path: /lens-ingest
pathType: Prefix
backend:
service:
name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }}
name: {{ .Values.lens.service.name }}
port:
number: {{ .Values.lensWorker.service.port }}
number: {{ .Values.lens.service.port }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }}
{{- end }}

View file

@ -1,3 +0,0 @@
{{- if eq (include "litellm.lens.mode" .) "bundled" }}
{{ include "litellm.lens.render" (dict "root" . "resource" "service") }}
{{- end }}

View file

@ -15,7 +15,6 @@ tests:
ingress.componentHosts.ui: ui.example.com
ingress.componentHosts.backend: backend.example.com
ingress.componentHosts.gateway: gateway.example.com
lensWorker.mode: disabled
ingress.tls:
- hosts:
- ui.example.com
@ -83,12 +82,12 @@ tests:
ui.backendUrl: https://backend.example.com
ui.publicUrl: https://ui.example.com
backend.gatewayUrl: https://gateway.example.com
lensWorker.mode: external
lensWorker.externalServiceName: existing-lens
lensWorker.externalUrl: https://lens.example.com
lensWorker.publicUrl: https://lens.example.com
lensWorker.serviceTokenSecret.name: existing-lens-signing
lensWorker.gateway.secretName: existing-lens-signing
lens.enabled: true
lens.service.name: existing-lens
lens.url: http://existing-lens:4318
lens.publicUrl: https://gateway.example.com/lens-ingest
lens.serviceTokenSecret.name: existing-lens-signing
lens.gatewaySecret.name: existing-lens-signing
asserts:
- equal:
path: spec.rules[2].http.paths[0].path

View file

@ -0,0 +1,166 @@
suite: lens connection
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
- gateway/configmap.yaml
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: connects the gateway to an external Lens
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: connects the backend to an external Lens
template: backend/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: gateway/deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service on the single ingress host
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
ingress.host: litellm.example.com
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

View file

@ -1,86 +0,0 @@
suite: Lens endpoint defaults
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
set:
lensWorker.enabled: true
ingress.enabled: true
ingress.host: gateway.example
ingress.tls:
- hosts: [gateway.example]
secretName: tls
tests:
- it: derives the trace endpoint from the deployment hostname and TLS
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- it: preserves an explicitly configured public address
set:
lensWorker.publicUrl: https://custom.example/prefix
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://custom.example/prefix
- it: uses the dedicated Lens ingress when configured
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls:
- hosts: [traces.example]
secretName: traces-tls
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTPS for a dedicated ALB ingress with certificate annotations
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.className: alb
lensWorker.ingress.tls: []
lensWorker.ingress.annotations:
alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]'
alb.ingress.kubernetes.io/ssl-redirect: "443"
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://traces.example
- it: uses HTTP for a dedicated ingress without its own TLS
set:
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
lensWorker.ingress.tls: []
lensWorker.ingress.annotations: {}
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://traces.example
- it: uses HTTP when ingress has no TLS
set:
ingress.tls: []
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: http://gateway.example/lens-ingest
values:
- ./values/required.yaml

View file

@ -1,208 +0,0 @@
---
suite: Lens deployment modes
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
- ingress.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: retains the existing deployment before transferring its release ownership
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.retainResources: true
asserts:
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: keeps the ingestion endpoint on an independently deployed Lens service
template: ingress.yaml
set:
ingress.enabled: true
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: existing-lens
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: connects an external deployment without installing Lens resources
templates:
- lens/deployment.yaml
- lens/service.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- hasDocuments:
count: 0
- it: sends delegated requests to the explicitly selected external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.other-namespace:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: external-lens-signing
key: gateway-secret
- it: disables Lens explicitly even with a retained legacy enable flag
template: lens/deployment.yaml
set:
lensWorker.mode: disabled
lensWorker.enabled: true
asserts:
- hasDocuments:
count: 0
- it: bundles Lens through explicit mode selection
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
asserts:
- hasDocuments:
count: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_MODE
value: standalone
- it: uses separate credentials for external identity and service access
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.serviceTokenSecret: {name: worker-service, key: token}
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef: {name: worker-service, key: token}
- it: supplies the same separate signing secret to the bundled service
template: lens/deployment.yaml
set:
lensWorker.mode: bundled
lensWorker.gateway: {secretName: delegated-identity, secretKey: signature}
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: delegated-identity, key: signature}
- it: routes explicit bundled mode with the enable flag off
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: bundled
lensWorker.enabled: false
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: gateway-lens-worker
port: {number: 4318}
- it: routes external mode to its selected service
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: "existing-lens"
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port: {number: 4318}
- it: omits ingestion when an external service has no ingress backend
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: external
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true
- it: omits ingestion for disabled mode with the enable flag on
template: ingress.yaml
set:
fullnameOverride: gateway
ingress.enabled: true
ingress.host: gateway.example
lensWorker.mode: disabled
lensWorker.enabled: true
lensWorker.externalUrl: http://existing-lens:4318
lensWorker.externalServiceName: ""
lensWorker.serviceTokenSecret.name: worker-service
lensWorker.gateway.secretName: external-lens-signing
asserts:
- notContains:
path: spec.rules[0].http.paths
content: {path: /lens-ingest}
any: true

View file

@ -1,54 +0,0 @@
---
suite: Lens deployment mode validation
templates:
- gateway/configmap.yaml
- backend/deployment.yaml
values:
- "./values/required.yaml"
set:
lensWorker.publicUrl: https://traces.example
tests:
- it: requires a separate identity signing reference for external Lens
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: worker-service
asserts:
- failedTemplate:
errorMessage: lensWorker.gateway.secretName is required for external Lens
- it: requires an external service address
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
asserts:
- failedTemplate:
errorMessage: lensWorker.externalUrl is required for external Lens
- it: requires the shared server credential for an external service
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
asserts:
- failedTemplate:
errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens
- it: requires an agent reachable URL for external ingestion
template: backend/deployment.yaml
set:
lensWorker.mode: external
lensWorker.externalUrl: http://lens.other-namespace:4318
lensWorker.serviceTokenSecret.name: external-lens-signing
lensWorker.gateway.secretName: external-lens-signing
lensWorker.publicUrl: ''
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required for external Lens
- it: rejects unsupported deployment modes
template: backend/deployment.yaml
set:
lensWorker.mode: typo
asserts:
- failedTemplate:
errorMessage: lensWorker.mode must be bundled, external or disabled

View file

@ -1,61 +0,0 @@
suite: Lens preserves credentials across Helm upgrades
templates:
- lens/secrets.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
release:
namespace: lens
name: lens-test
upgrade: true
kubernetesProvider:
scheme:
v1/Secret:
gvr:
version: v1
resource: secrets
namespaced: true
objects:
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-gateway
namespace: lens
data:
gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk=
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-service
namespace: lens
data:
service-token: c2F2ZWQtc2VydmljZS10b2tlbg==
- apiVersion: v1
kind: Secret
metadata:
name: lens-test-lens-clickhouse
namespace: lens
data:
password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
tests:
- it: preserves the separate signing credential across upgrades
documentSelector: {path: metadata.name, value: lens-test-lens-gateway}
asserts:
- equal:
path: data.gateway-secret
value: c2F2ZWQtc2lnbmluZy1rZXk=
- notExists: {path: data.service-token}
- it: reuses the service credential instead of breaking running services
documentIndex: 0
asserts:
- equal:
path: data.service-token
value: c2F2ZWQtc2VydmljZS10b2tlbg==
- it: reuses the database password instead of locking out stored traces
documentIndex: 1
asserts:
- equal:
path: data.password
value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ=
values:
- ./values/required.yaml

View file

@ -1,196 +0,0 @@
suite: Lens service isolation and ingestion routing
templates:
- gateway/configmap.yaml
- gateway/deployment.yaml
- backend/deployment.yaml
- ingress.yaml
- lens/ingress.yaml
- lens/service.yaml
- lens/deployment.yaml
tests:
- it: connects gateway/deployment.yaml to the shared Lens service
template: gateway/deployment.yaml
set: &id001
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: connects backend/deployment.yaml to the shared Lens service
template: backend/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens-test-lens-worker:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://gateway.example/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-test-lens-gateway
key: gateway-secret
- it: routes uploads directly to Lens instead of the gateway
template: ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
ingress.enabled: true
ingress.host: gateway.example
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
number: 4318
- it: keeps internal routes out of a dedicated ingestion hostname
template: lens/ingress.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://gateway.example/lens-ingest
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.ingress.enabled: true
lensWorker.ingress.host: traces.example
asserts:
- equal:
path: spec.rules[0].http.paths
value:
- path: /v1/
pathType: Prefix
backend:
service:
name: lens-test-lens-worker
port:
name: otlp
- it: maps the Lens service to the ingestion listener
template: lens/service.yaml
set: *id001
asserts:
- equal:
path: spec.selector
value:
app.kubernetes.io/instance: RELEASE-NAME
app.kubernetes.io/component: lens-worker
- equal:
path: spec.ports
value:
- name: otlp
port: 4318
targetPort: otlp
- it: gives only Lens the ClickHouse secret
template: lens/deployment.yaml
set: *id001
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_URL
valueFrom:
secretKeyRef:
name: lens-storage
key: url
- it: requires an agent reachable ingestion URL
template: gateway/deployment.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: ''
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- failedTemplate:
errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname
- it: omits Lens connection settings when disabled in gateway/deployment.yaml
template: gateway/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: omits Lens connection settings when disabled in backend/deployment.yaml
template: backend/deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
any: true
- it: preserves an existing ClickHouse database and retention
template: lens/deployment.yaml
set:
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
lensWorker.clickhouseDatabase: existing_traces
lensWorker.retentionDays: 45
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_DATABASE
value: existing_traces
- contains:
path: spec.template.spec.containers[0].env
content:
name: AGENT_TRACING_RETENTION_DAYS
value: '45'
- it: keeps Lens pods outside the gateway autoscaling selector
template: lens/deployment.yaml
set: &id002
nameOverride: inference
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
asserts:
- equal:
path: spec.template.metadata.labels["app.kubernetes.io/name"]
value: inference-lens-worker
- it: preserves the existing gateway deployment selector
template: gateway/deployment.yaml
set: *id002
asserts:
- equal:
path: spec.selector.matchLabels["app.kubernetes.io/name"]
value: inference
values:
- ./values/required.yaml

View file

@ -1,164 +0,0 @@
suite: Lens managed setup
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
release:
namespace: lens
name: lens-test
templates:
- lens/deployment.yaml
- lens/clickhouse.yaml
- lens/secrets.yaml
tests:
- it: generates private login, gateway and storage credentials
template: lens/secrets.yaml
asserts:
- hasDocuments:
count: 4
- matchRegex:
path: data.service-token
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 0
- matchRegex:
path: data.password
pattern: '^[A-Za-z0-9+/]{86}==$'
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/resource-policy"]
value: keep
- it: connects Lens to its private bundled storage
template: lens/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: lens-test-lens-clickhouse
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: stores traces on a persistent volume with the chosen storage class
template: lens/clickhouse.yaml
documentIndex: 1
set:
lensWorker.clickhouse.storage: 40Gi
lensWorker.clickhouse.storageClassName: fast
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec
value:
accessModes: [ReadWriteOnce]
storageClassName: fast
resources:
requests:
storage: 40Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: lens-test-lens-clickhouse
key: password
- it: keeps an existing external database instead of creating a new one
template: lens/clickhouse.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
asserts:
- hasDocuments:
count: 0
- it: leaves supplied secrets under their existing manager
template: lens/secrets.yaml
set:
lensWorker.clickhouseSecret.name: external-clickhouse
lensWorker.serviceTokenSecret.name: external-service
lensWorker.adminTokenSecret.name: external-admin
lensWorker.gateway.secretName: external-signing
asserts:
- hasDocuments:
count: 0
- it: creates no credentials or database when Lens is disabled
templates:
- lens/secrets.yaml
- lens/clickhouse.yaml
set:
lensWorker.enabled: false
asserts:
- hasDocuments:
count: 0
- it: requires an external database when bundled storage is explicitly disabled
template: lens/deployment.yaml
set:
lensWorker.clickhouse.enabled: false
asserts:
- failedTemplate:
errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled
- it: keeps storage names valid and uses the same name for the connection
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 0
- matchRegex:
path: metadata.name
pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$'
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 0
- equal:
path: spec.serviceName
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/clickhouse.yaml
documentIndex: 1
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/secrets.yaml
documentIndex: 1
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_HOST
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
template: lens/deployment.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: CLICKHOUSE_PASSWORD
valueFrom:
secretKeyRef:
name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse
key: password
template: lens/deployment.yaml
- it: allows cold storage startup and inherits registry credentials
template: lens/clickhouse.yaml
documentIndex: 1
set:
imagePullSecrets: [{name: registry-auth}]
asserts:
- equal:
path: spec.template.spec.imagePullSecrets
value: [{name: registry-auth}]
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet: {path: /ping, port: http}
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 60
values:
- ./values/required.yaml

View file

@ -1,79 +0,0 @@
suite: Independent Lens release and credentials
templates:
- lens/deployment.yaml
values:
- ./values/required.yaml
set:
fullnameOverride: lens-test
lensWorker.enabled: true
lensWorker.publicUrl: https://traces.example
lensWorker.serviceTokenSecret.name: lens-service
lensWorker.clickhouseSecret.name: lens-storage
tests:
- it: selects a Lens release independently of the gateway
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
lensWorker.image.tag: 2.3.4
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:2.3.4
- it: retains the pinned Lens chart default when only the gateway changes
chart:
appVersion: 9.8.7
set:
backend.image.tag: gateway-9.8.7
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens:0.1.0-dev.0
- it: selects development publications explicitly
set:
lensWorker.image.repository: ghcr.io/berriai/lens-dev
lensWorker.image.tag: sha-0123456789abcdef
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef
- it: preserves a verified registry mirror and digest
set:
lensWorker.image.repository: registry.example/lens
lensWorker.image.tag: replaced-label
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- it: rejects a malformed digest
set:
lensWorker.image.digest: sha256:invalid
asserts:
- failedTemplate:
errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters
- it: keeps the legacy enablement switch working
set:
lensWorker.enabled: false
asserts:
- hasDocuments: {count: 0}
- it: keeps isolated local analysis with delegated gateway access
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: LENS_MODE, value: standalone}
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret}
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
- equal:
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
value: true
- equal:
path: spec.template.spec.volumes[0].emptyDir
value: {medium: Memory, sizeLimit: 1Gi}

View file

@ -654,70 +654,15 @@ ui:
topologySpreadConstraints: []
lens:
library: true
lensWorker:
retainResources: false
mode: ""
externalUrl: ""
externalServiceName: ""
gateway:
secretName: ""
secretKey: gateway-secret
standaloneUrl: ""
adminTokenSecret:
name: ""
key: admin-token
extraEnv: []
extraEnvFrom: []
enabled: false
replicaCount: 1
image:
repository: ghcr.io/berriai/lens
tag: "0.1.0-dev.0"
digest: ""
pullPolicy: IfNotPresent
tokenSecret:
url: ""
publicUrl: ""
service:
name: ""
key: token
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token
clickhouse:
enabled: true
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
storage: 20Gi
storageClassName: null
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
memory: 2Gi
clickhouseDatabase: litellm
retentionDays: 14
clickhouseSecret:
name: ""
key: url
publicUrl: ""
service:
port: 4318
annotations: {}
ingress:
enabled: false
path: /v1/
className: ""
host: ""
annotations: {}
tls: []
url: ""
tmpSizeLimit: 1Gi
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 2Gi
nodeSelector: {}
tolerations: []
affinity: {}

View file

@ -1,477 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
qualification_mode=${LENS_QUALIFICATION_MODE:-smoke}
chart_selection=${LENS_HELM_CHART:-all}
case "$qualification_mode" in smoke|boundaries|release) ;; *) printf 'LENS_QUALIFICATION_MODE must be smoke, boundaries or release\n' >&2; exit 2 ;; esac
case "$chart_selection" in all|litellm-helm|litellm) ;; *) printf 'LENS_HELM_CHART must be all, litellm-helm or litellm\n' >&2; exit 2 ;; esac
if [[ "$qualification_mode" == release ]]; then
: "${LENS_RELEASE_PROVIDER_API_KEY:?A separately authorized provider credential is required}"
: "${LENS_RELEASE_PROVIDER_API_BASE:?Set the authorized provider API base}"
: "${LENS_RELEASE_PROVIDER_MODEL:?Set the provider model explicitly}"
[[ "$LENS_RELEASE_PROVIDER_API_BASE" == https://* ]]
fi
qa_dir=$(mktemp -d)
cluster="lens-install-$(openssl rand -hex 6)"
cluster_owned=false
forward_pids=()
cleanup() {
local status=$?
if declare -F qualification_finish > /dev/null; then qualification_finish "$status" || true; fi
if (( status != 0 )); then
for log in "$qa_dir"/*-forward.log; do
if [[ -f "$log" ]]; then cat "$log" >&2; fi
done
if [[ -n "${namespace:-}" ]]; then diagnose || true; fi
fi
for pid in "${forward_pids[@]}"; do kill "$pid" 2>/dev/null || true; done
if [[ "$cluster_owned" == true ]]; then kind delete cluster --name "$cluster" || true; fi
rm -rf "$qa_dir"
return "$status"
}
trap cleanup EXIT
umask 077
export KUBECONFIG="$qa_dir/kubeconfig"
for component in gateway backend monolith; do
baseline_id=$(docker image inspect "lens-ci-$component:v0.0.0-lens-ci-baseline" --format '{{.Id}}')
current_id=$(docker image inspect "lens-ci-$component:v0.0.0-lens-ci" --format '{{.Id}}')
test "$baseline_id" != "$current_id"
done
cluster_owned=true
kind create cluster --name "$cluster" \
--image kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f \
--wait 120s
for component in gateway backend ui migrations monolith; do
kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci"
done
for component in gateway backend monolith; do
kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci-baseline"
done
test "$(docker image inspect lens-ci-worker:baseline --format '{{.Id}}')" != \
"$(docker image inspect lens-ci-worker:upgrade --format '{{.Id}}')"
kind load docker-image --name "$cluster" lens-ci-worker:baseline lens-ci-worker:upgrade
test -f helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz
api() {
curl --fail-with-body --silent --show-error --max-time 20 \
-H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \
"http://127.0.0.1:14418$1" "${@:2}"
}
saved_trace() {
for attempt in $(seq 1 30); do
if api "/v1/traces/$trace_id" > "$qa_dir/saved.json" && \
jq -e --arg span "$span_id" 'any(.spans[]; .span_id == $span)' "$qa_dir/saved.json" > /dev/null; then
return 0
fi
sleep 2
done
return 1
}
diagnose() {
kubectl -n "$namespace" get pods
kubectl -n "$namespace" get services,endpoints
kubectl -n "$namespace" get events --sort-by=.lastTimestamp | tail -30
if [[ "$qualification_mode" == smoke ]]; then
kubectl -n "$namespace" logs --all-containers -l app.kubernetes.io/instance=lens --tail=50 || true
fi
return 1
}
forward() {
local service=$1 local_port=$2 remote_port=$3
local log="$qa_dir/$service-forward.log"
kubectl -n "$namespace" port-forward --address 127.0.0.1 --pod-running-timeout=30s \
"service/$service" "$local_port:$remote_port" > "$log" 2>&1 &
local pid=$!
forward_pids+=("$pid")
for attempt in $(seq 1 150); do
if ! kill -0 "$pid" 2>/dev/null; then
cat "$log" >&2
return 1
fi
if grep -q "^Forwarding from 127\\.0\\.0\\.1:$local_port ->" "$log"; then return 0; fi
sleep 0.2
done
cat "$log" >&2
return 1
}
stop_forwards() {
for pid in "${forward_pids[@]}"; do
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
done
forward_pids=()
}
forward_bundled() {
stop_forwards
qualification_forward_control
forward lens-lens-worker 14419 4318
}
deployment_pods() {
local deployment selector deployment_uid replica_sets attempt
local pods="$qa_dir/$1-pods.json" snapshot="$qa_dir/$1-snapshot.json"
printf '%s: wait for deployment/%s before recording pod identities\n' "$namespace" "$1" >&2
kubectl -n "$namespace" rollout status "deployment/$1" --timeout=180s >&2 || return
deployment=$(kubectl -n "$namespace" get deployment "$1" -o json) || return
selector=$(jq -r '.spec.selector.matchLabels | to_entries | map("\(.key)=\(.value)") | join(",")' <<< "$deployment")
deployment_uid=$(jq -er .metadata.uid <<< "$deployment")
test -n "$selector"
replica_sets=$(kubectl -n "$namespace" get replicasets -l "$selector" -o json \
| jq -ce --arg uid "$deployment_uid" '[.items[]
| select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "Deployment" and .uid == $uid))
| .metadata.uid] | if length > 0 then . else error("Deployment ReplicaSet identity is missing") end') || return
for attempt in $(seq 1 30); do
kubectl -n "$namespace" get pods -l "$selector" -o json > "$pods" || return
if jq -S -e --argjson replica_sets "$replica_sets" '[.items[]
| select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "ReplicaSet"
and (.uid as $owner | $replica_sets | index($owner) != null)))
| select(.metadata.deletionTimestamp == null)
| {uid:.metadata.uid, images:.spec.containers | map({name,image}),
containers:(.status.containerStatuses // []) | map({name,imageID,restartCount,ready})}] | sort_by(.uid)
| if length > 0 and all(.[]; .containers | length > 0)
and all(.[].containers[]; .imageID != null and .imageID != "" and .ready)
then . else error("Ready pod runtime identity is missing") end' \
"$pods" > "$snapshot" 2> "$qa_dir/$1-snapshot-error.log"; then
cat "$snapshot"
return 0
fi
sleep 1
done
printf '%s: deployment/%s pod identities did not converge\n' "$namespace" "$1" >&2
jq '[.items[] | {name:.metadata.name, uid:.metadata.uid, deleting:.metadata.deletionTimestamp,
phase:.status.phase, containers:[.status.containerStatuses[]? | {name,imageID,restartCount,ready}]}]' \
"$pods" >&2
return 1
}
gateway_pods() {
local component=$1 tag=$2 deployment="lens-$1"
if [[ "$component" == monolith ]]; then deployment=lens; fi
deployment_pods "$deployment" \
| jq -S -e --arg expected "lens-ci-$component:$tag" '
if all(.[]; any(.images[]; .image == $expected)) then
[.[] as $pod | $pod.images[] | select(.image == $expected) | . as $requested
| $pod.containers[] | select(.name == $requested.name)
| {uid:$pod.uid, image:$requested.image, name, imageID, restartCount, ready}]
else error("Gateway pod does not use the expected image") end'
}
migration_job() {
kubectl -n "$namespace" get job lens-migrations -o json \
| jq -e --arg image "$1" '
select(.metadata.annotations["helm.sh/hook"] == "pre-install,pre-upgrade")
| select(.metadata.annotations["helm.sh/hook-delete-policy"] == "before-hook-creation")
| select(.metadata.annotations["argocd.argoproj.io/hook"] == null)
| select(.status.succeeded == 1 and .spec.template.spec.containers[0].image == $image)
| .metadata.uid | select(type == "string" and length > 0)'
}
source tests/e2e/migrations/lens_release_qualification.sh
qualification_initialize
if [[ "$qualification_mode" != smoke && "$chart_selection" != litellm ]]; then
qualification_bundled_postgres
fi
for chart in litellm-helm litellm; do
if [[ "$chart_selection" != all && "$chart_selection" != "$chart" ]]; then continue; fi
namespace="lens-$chart"
kubectl create namespace "$namespace"
master_key="sk-$(openssl rand -hex 24)"
kubectl -n "$namespace" create secret generic lens-secrets \
--from-literal="master-key=$master_key" \
--from-literal="service-token=$(openssl rand -hex 32)" \
--from-literal="gateway-secret=$(openssl rand -hex 32)" \
--from-literal="url=http://clickhouse:8123" \
--from-literal=username=litellm --from-literal=password=isolated-helm-test
kubectl -n "$namespace" apply -f - <<'YAML'
apiVersion: apps/v1
kind: Deployment
metadata: {name: postgres}
spec:
selector: {matchLabels: {app: postgres}}
template:
metadata: {labels: {app: postgres}}
spec:
containers:
- name: postgres
image: postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
env:
- {name: POSTGRES_DB, value: litellm}
- {name: POSTGRES_USER, value: litellm}
- {name: POSTGRES_PASSWORD, value: isolated-helm-test}
readinessProbe:
exec: {command: [pg_isready, -U, litellm, -d, litellm]}
---
apiVersion: v1
kind: Service
metadata: {name: postgres}
spec:
selector: {app: postgres}
ports: [{port: 5432}]
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: clickhouse}
spec:
selector: {matchLabels: {app: clickhouse}}
template:
metadata: {labels: {app: clickhouse}}
spec:
containers:
- name: clickhouse
image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
env: [{name: CLICKHOUSE_SKIP_USER_SETUP, value: "1"}]
volumeMounts:
- {name: keeper, mountPath: /etc/clickhouse-server/config.d/lens-keeper.xml, subPath: lens-keeper.xml}
readinessProbe:
httpGet: {path: /ping, port: 8123}
volumes:
- name: keeper
configMap: {name: clickhouse-keeper}
---
apiVersion: v1
kind: ConfigMap
metadata: {name: clickhouse-keeper}
data:
lens-keeper.xml: |
<clickhouse>
<keeper_server>
<tcp_port>9181</tcp_port><server_id>1</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<coordination_settings><operation_timeout_ms>10000</operation_timeout_ms><quorum_reads>true</quorum_reads></coordination_settings>
<raft_configuration><server><id>1</id><hostname>127.0.0.1</hostname><port>9234</port></server></raft_configuration>
</keeper_server>
<zookeeper><node><host>127.0.0.1</host><port>9181</port></node></zookeeper>
<keeper_map_path_prefix>/lens/keeper-map</keeper_map_path_prefix>
</clickhouse>
---
apiVersion: v1
kind: Service
metadata: {name: clickhouse}
spec:
selector: {app: clickhouse}
ports: [{port: 8123}]
YAML
kubectl -n "$namespace" rollout status deployment/postgres --timeout=180s
kubectl -n "$namespace" rollout status deployment/clickhouse --timeout=180s
cat > "$qa_dir/common.yaml" <<'YAML'
fullnameOverride: lens
migrationJob:
ttlSecondsAfterFinished: 3600
hooks:
helm: {enabled: true}
argocd: {enabled: false}
lensWorker:
enabled: true
image: {repository: lens-ci-worker, tag: baseline, pullPolicy: Never}
serviceTokenSecret: {name: lens-secrets, key: service-token}
gateway: {secretName: lens-secrets, secretKey: gateway-secret}
clickhouseSecret: {name: lens-secrets, key: url}
clickhouseDatabase: existing_traces
retentionDays: 45
publicUrl: http://127.0.0.1:14419
YAML
if [[ "$chart" == litellm-helm ]]; then
control=lens
control_port=4000
cat > "$qa_dir/chart.yaml" <<'YAML'
image: {repository: lens-ci-monolith, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never}
masterkeySecretName: lens-secrets
masterkeySecretKey: master-key
envVars: {STORE_MODEL_IN_DB: "True"}
db:
deployStandalone: false
useExisting: true
endpoint: postgres
secret: {name: lens-secrets, usernameKey: username, passwordKey: password}
redis: {enabled: false}
proxy_config:
model_list: []
general_settings:
master_key: os.environ/PROXY_MASTER_KEY
store_model_in_db: true
tracing: {enabled: true, store: {type: lens}}
YAML
else
control=lens-backend
control_port=4001
cat > "$qa_dir/chart.yaml" <<'YAML'
masterKey: {secretName: lens-secrets, secretKey: master-key}
database:
writer:
host: postgres
dbname: litellm
passwordSecret: {name: lens-secrets, usernameKey: username, passwordKey: password}
migrationJob:
image: {repository: lens-ci-migrations, tag: v0.0.0-lens-ci, pullPolicy: Never}
gateway:
image: {repository: lens-ci-gateway, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never}
numWorkers: 1
extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}]
hpa: {enabled: false}
resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}}
config:
create: true
proxy_config:
model_list: []
general_settings:
store_model_in_db: true
tracing: {enabled: true, store: {type: lens}}
backend:
extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}]
image: {repository: lens-ci-backend, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never}
hpa: {enabled: false}
resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}}
ui:
image: {repository: lens-ci-ui, tag: v0.0.0-lens-ci, pullPolicy: Never}
hpa: {enabled: false}
YAML
fi
install=(helm upgrade --install lens "helm/$chart" -n "$namespace" \
-f "$qa_dir/common.yaml" -f "$qa_dir/chart.yaml" --wait --wait-for-jobs --timeout 8m)
qualification_provider_values
"${install[@]}" || diagnose
forward_bundled
for attempt in $(seq 1 30); do
if api /lens/service > "$qa_dir/status.json" && jq -e '.connected and .status.storage_ready' "$qa_dir/status.json"; then break; fi
sleep 1
done
jq -e '.connected and .status.storage_ready' "$qa_dir/status.json"
qualification_setup
api /lens/tracing/keys -d '{"name":"Helm smoke"}' > "$qa_dir/key.json"
tracing_key=$(jq -r .key "$qa_dir/key.json")
trace_id=$(openssl rand -hex 16)
span_id=$(openssl rand -hex 8)
jq -n --arg trace "$trace_id" --arg span "$span_id" --arg at "$(date +%s)000000000" \
'{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Helm trace",kind:1,
startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/trace.json"
curl --fail-with-body --silent --show-error --retry 10 --retry-all-errors --retry-delay 1 \
-H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \
-d "@$qa_dir/trace.json" http://127.0.0.1:14419/v1/traces
saved_trace
qualification_read_boundaries baseline
kubectl -n "$namespace" exec deployment/clickhouse -- clickhouse-client --query \
"SELECT count() FROM existing_traces.otel_traces WHERE TraceId = '$trace_id'" | grep -qx 1
"${install[@]}" || diagnose
forward_bundled
saved_trace
qualification_read_boundaries reapplied
kubectl -n "$namespace" get deployments -l app.kubernetes.io/instance=lens -o json \
| jq -S '[.items[] | select(.metadata.name != "lens-lens-worker") | {name:.metadata.name,template:.spec.template}] | sort_by(.name)' \
> "$qa_dir/gateway-before.json"
"${install[@]}" --set lensWorker.image.tag=upgrade || diagnose
kubectl -n "$namespace" get deployments -l app.kubernetes.io/instance=lens -o json \
| jq -S '[.items[] | select(.metadata.name != "lens-lens-worker") | {name:.metadata.name,template:.spec.template}] | sort_by(.name)' \
> "$qa_dir/gateway-after.json"
cmp "$qa_dir/gateway-before.json" "$qa_dir/gateway-after.json"
forward_bundled
saved_trace
qualification_read_boundaries lens-upgrade
"${install[@]}" || diagnose
forward_bundled
saved_trace
qualification_read_boundaries lens-rollback
kubectl -n "$namespace" get deployment lens-lens-worker -o json \
| jq -S .spec.template > "$qa_dir/lens-before.json"
deployment_pods lens-lens-worker > "$qa_dir/lens-pods-before.json"
gateway_components=(monolith)
gateway_upgrade=(--set image.tag=v0.0.0-lens-ci)
migration_baseline=lens-ci-monolith:v0.0.0-lens-ci-baseline
migration_current=lens-ci-monolith:v0.0.0-lens-ci
if [[ "$chart" == litellm ]]; then
gateway_components=(gateway backend)
gateway_upgrade=(--set gateway.image.tag=v0.0.0-lens-ci --set backend.image.tag=v0.0.0-lens-ci)
migration_baseline=lens-ci-migrations:v0.0.0-lens-ci
migration_current=$migration_baseline
fi
migration_before=$(migration_job "$migration_baseline")
for component in "${gateway_components[@]}"; do
gateway_pods "$component" v0.0.0-lens-ci-baseline > "$qa_dir/$component-before.json"
done
install+=("${gateway_upgrade[@]}")
"${install[@]}" || diagnose
migration_after=$(migration_job "$migration_current")
test "$migration_before" != "$migration_after"
kubectl -n "$namespace" get deployment lens-lens-worker -o json \
| jq -S .spec.template > "$qa_dir/lens-after.json"
cmp "$qa_dir/lens-before.json" "$qa_dir/lens-after.json"
deployment_pods lens-lens-worker > "$qa_dir/lens-pods-after.json"
cmp "$qa_dir/lens-pods-before.json" "$qa_dir/lens-pods-after.json"
for component in "${gateway_components[@]}"; do
gateway_pods "$component" v0.0.0-lens-ci > "$qa_dir/$component-after.json"
before_id=$(jq -ce '[.[].imageID] | unique | if length == 1 then .[0] else error("Mixed gateway images") end' \
"$qa_dir/$component-before.json")
after_id=$(jq -ce '[.[].imageID] | unique | if length == 1 then .[0] else error("Mixed gateway images") end' \
"$qa_dir/$component-after.json")
test "$before_id" != "$after_id"
done
forward_bundled
saved_trace
qualification_read_boundaries gateway-upgrade
if [[ "$qualification_mode" != smoke ]]; then
install+=(--set lensWorker.image.tag=upgrade)
"${install[@]}" || diagnose
forward_bundled
qualification_read_boundaries both-candidates
fi
stop_forwards
kubectl -n "$namespace" rollout restart "deployment/$control" deployment/lens-lens-worker
kubectl -n "$namespace" rollout status "deployment/$control" --timeout=180s
kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s
qualification_forward_control
saved_trace
qualification_outage
printf '%s: fresh install, ingestion, Lens upgrade and rollback, independent gateway image upgrade, and restart passed\n' "$chart"
stop_forwards
helm upgrade --install external-lens helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz \
-n "$namespace" --wait --timeout 5m \
--set fullnameOverride=external-lens \
--set image.repository=lens-ci-worker --set image.tag=upgrade --set image.pullPolicy=Never \
--set adminTokenSecret.name=lens-secrets --set adminTokenSecret.key=master-key \
--set gateway.enabled=true --set gateway.secretName=lens-secrets \
--set gateway.secretKey=gateway-secret --set serviceTokenSecret.name=lens-secrets \
--set clickhouseSecret.name=lens-secrets --set clickhouseDatabase=existing_traces \
--set publicUrl=http://127.0.0.1:14419
"${install[@]}" --set lensWorker.mode=external \
--set lensWorker.externalUrl=http://external-lens:4318 || diagnose
test -z "$(kubectl -n "$namespace" get deployment lens-lens-worker --ignore-not-found -o name)"
qualification_forward_control
forward external-lens 14419 4318
api /lens/service | jq -e '.configured and .connected and .status.storage_ready'
saved_trace
qualification_read_boundaries external-lens
trace_id=$(openssl rand -hex 16)
span_id=$(openssl rand -hex 8)
jq --arg trace "$trace_id" --arg span "$span_id" \
'.resourceSpans[0].scopeSpans[0].spans[0] |= (.traceId=$trace | .spanId=$span)' \
"$qa_dir/trace.json" > "$qa_dir/external-trace.json"
curl --fail-with-body --silent --show-error --max-time 20 \
-H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \
-d "@$qa_dir/external-trace.json" http://127.0.0.1:14419/v1/traces
saved_trace
kubectl -n "$namespace" get deployment external-lens -o json \
| jq -S '{uid:.metadata.uid,template:.spec.template}' > "$qa_dir/external-before.json"
stop_forwards
tracing_setting=proxy_config.general_settings.tracing.enabled
if [[ "$chart" == litellm ]]; then
tracing_setting=gateway.config.proxy_config.general_settings.tracing.enabled
fi
"${install[@]}" --set lensWorker.mode=disabled --set "$tracing_setting=false" || diagnose
test -z "$(kubectl -n "$namespace" get deployment lens-lens-worker --ignore-not-found -o name)"
qualification_forward_control
api /lens/service | jq -e '.configured == false and .connected == false'
test "$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 20 \
-H "Authorization: Bearer $master_key" http://127.0.0.1:14418/lens/datasets)" = 503
kubectl -n "$namespace" get deployment external-lens -o json \
| jq -S '{uid:.metadata.uid,template:.spec.template}' > "$qa_dir/external-after.json"
cmp "$qa_dir/external-before.json" "$qa_dir/external-after.json"
printf '%s: external Lens trace reads/writes and disabled gateway behavior passed\n' "$chart"
qualification_record chart-lifecycle passed
stop_forwards
kubectl delete namespace "$namespace" --wait=true
done

View file

@ -1,284 +0,0 @@
#!/usr/bin/env bash
qualification_initialize() {
qualification_events="$qa_dir/qualification-events.jsonl"
: > "$qualification_events"
qualification_started=$(date -u +%FT%TZ)
qualification_output=${LENS_QUALIFICATION_RESULTS_DIR:-}
if [[ -n "$qualification_output" ]]; then
mkdir -p "$qualification_output"
test ! -e "$qualification_output/host-$chart_selection-$qualification_mode.json"
fi
}
qualification_record() {
if [[ "$qualification_mode" == smoke ]]; then return; fi
jq -nc --arg chart "${chart:-bootstrap}" --arg case "$1" --arg result "$2" \
--arg at "$(date -u +%FT%TZ)" '{chart:$chart,case:$case,result:$result,at:$at}' >> "$qualification_events"
}
qualification_finish() {
if [[ "$qualification_mode" == smoke || -z "$qualification_output" ]]; then return; fi
jq -n --slurpfile checks "$qualification_events" --arg mode "$qualification_mode" \
--arg selection "$chart_selection" --arg started "$qualification_started" \
--arg finished "$(date -u +%FT%TZ)" --arg source "$(git rev-parse HEAD)" --argjson exit_code "$1" \
'{qualification:"isolated candidate host behavior",mode:$mode,chart_selection:$selection,
started_at:$started,finished_at:$finished,harness_source:$source,exit_code:$exit_code,
passed:($exit_code == 0),paid_provider_qualified:($mode == "release" and $exit_code == 0),
signed_release_qualified:false,production_deployed:false,checks:$checks}' \
> "$qualification_output/host-$chart_selection-$qualification_mode.json"
}
qualification_request() {
local credential=$1 method=$2 route=$3 expected=$4 label=$5 input=${6:-}
local port=14418
if [[ "$chart" == litellm && "$route" == /v1/chat/completions ]]; then port=14420; fi
local response="$qa_dir/release-response.json" status
printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "$credential" > "$qa_dir/release-headers"
local arguments=(--silent --show-error --max-time 90 --request "$method" \
--header "@$qa_dir/release-headers" --output "$response" --dump-header "$qa_dir/release-response.headers" \
--write-out '%{http_code}')
if [[ -n "$input" ]]; then arguments+=(--data-binary "@$input"); fi
status=$(curl "${arguments[@]}" "http://127.0.0.1:$port$route")
if [[ "$status" != "$expected" ]]; then
qualification_record "$label" "failed-http-$status-expected-$expected"
printf '%s: expected HTTP %s, received %s\n' "$label" "$expected" "$status" >&2
return 1
fi
qualification_record "$label" "http-$status"
}
qualification_forward_control() {
forward "$control" 14418 "$control_port"
if [[ "$qualification_mode" == release && "$chart" == litellm ]]; then
forward lens-gateway 14420 4000
fi
}
qualification_provider_values() {
if [[ "$qualification_mode" != release ]]; then return; fi
printf '%s' "$LENS_RELEASE_PROVIDER_API_KEY" > "$qa_dir/provider-key"
kubectl -n "$namespace" create secret generic lens-release-provider \
--from-file="LENS_RELEASE_PROVIDER_API_KEY=$qa_dir/provider-key" > /dev/null
if [[ "$chart" == litellm-helm ]]; then
printf 'environmentSecrets: [lens-release-provider]\n' > "$qa_dir/provider.yaml"
else
cat > "$qa_dir/provider.yaml" <<'YAML'
gateway:
extraEnv:
- {name: STORE_MODEL_IN_DB, value: "True"}
- name: LENS_RELEASE_PROVIDER_API_KEY
valueFrom: {secretKeyRef: {name: lens-release-provider, key: LENS_RELEASE_PROVIDER_API_KEY}}
backend:
extraEnv:
- {name: STORE_MODEL_IN_DB, value: "True"}
- name: LENS_RELEASE_PROVIDER_API_KEY
valueFrom: {secretKeyRef: {name: lens-release-provider, key: LENS_RELEASE_PROVIDER_API_KEY}}
YAML
fi
install+=(-f "$qa_dir/provider.yaml")
}
qualification_setup() {
if [[ "$qualification_mode" == smoke ]]; then return; fi
local tenant user team credential trace span attempt status
for tenant in a b; do
user="lens-release-$tenant-$(openssl rand -hex 8)"
team="lens-release-team-$tenant-$(openssl rand -hex 8)"
jq -n --arg user "$user" '{user_id:$user,user_email:($user+"@example.test"),user_role:"internal_user",auto_create_key:false}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /user/new 200 "create-user-$tenant" "$qa_dir/request.json"
jq -e --arg user "$user" '.user_id == $user' "$qa_dir/release-response.json" > /dev/null
jq -n --arg user "$user" --arg team "$team" '{team_id:$team,team_alias:$team,members_with_roles:[{user_id:$user,role:"admin"}]}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /team/new 200 "create-team-$tenant" "$qa_dir/request.json"
jq -e --arg team "$team" '.team_id == $team' "$qa_dir/release-response.json" > /dev/null
jq -n --arg user "$user" --arg team "$team" '{user_id:$user,team_id:$team,key_alias:($team+"-isolation")}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /key/generate 200 "create-key-$tenant" "$qa_dir/request.json"
credential=$(jq -er '.key | select(type == "string" and length > 0)' "$qa_dir/release-response.json")
jq -n --arg user "$user" --arg team "$team" --arg key "$credential" '{user:$user,team:$team,key:$key}' > "$qa_dir/tenant-$tenant.json"
jq -n --arg team "$team" '{name:"Release tenant trace",team_id:$team}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /lens/tracing/keys 200 "create-ingestion-key-$tenant" "$qa_dir/request.json"
cp "$qa_dir/release-response.json" "$qa_dir/ingestion-$tenant.json"
jq -e --arg team "$team" '.record.tenant.team_id == $team' "$qa_dir/ingestion-$tenant.json" > /dev/null
credential=$(jq -er .key "$qa_dir/ingestion-$tenant.json")
trace=$(openssl rand -hex 16)
span=$(openssl rand -hex 8)
jq -n --arg trace "$trace" --arg span "$span" --arg at "$(date +%s)000000000" \
'{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Lens release tenant",kind:1,startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/tenant-$tenant-trace.json"
printf '%s' "$trace" > "$qa_dir/tenant-$tenant-trace-id"
printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "$credential" > "$qa_dir/ingestion-headers"
for attempt in $(seq 1 30); do
status=$(curl --silent --show-error --max-time 20 --output "$qa_dir/ingestion-response.json" --write-out '%{http_code}' \
--header "@$qa_dir/ingestion-headers" --data-binary "@$qa_dir/tenant-$tenant-trace.json" http://127.0.0.1:14419/v1/traces)
if [[ "$status" == 200 ]]; then break; fi
test "$status" = 401
sleep 1
done
test "$status" = 200
qualification_record "ingest-team-$tenant" http-200
for attempt in $(seq 1 30); do
if api "/v1/traces/$trace" > "$qa_dir/tenant-$tenant-stored.json" 2> /dev/null; then break; fi
sleep 1
done
jq -e --arg trace "$trace" '.summary.trace_id == $trace' "$qa_dir/tenant-$tenant-stored.json" > /dev/null
done
jq -n --arg user "$(jq -r .user "$qa_dir/tenant-a.json")" --arg team "$(jq -r .team "$qa_dir/tenant-a.json")" \
'{user_id:$user,team_id:$team,key_alias:"Lens release revocation"}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /key/generate 200 create-revocation-key "$qa_dir/request.json"
cp "$qa_dir/release-response.json" "$qa_dir/revoked-key.json"
credential=$(jq -er .key "$qa_dir/revoked-key.json")
qualification_request "$credential" GET /lens/service 200 key-valid-before-revocation
jq -n --arg key "$credential" '{keys:[$key]}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /key/delete 200 revoke-gateway-key "$qa_dir/request.json"
qualification_request "$credential" GET /lens/service 401 revoked-key-denied
if [[ "$qualification_mode" == release ]]; then
qualification_model="lens-release-$(openssl rand -hex 8)"
jq -n --arg model "$qualification_model" --arg provider_model "$LENS_RELEASE_PROVIDER_MODEL" \
--arg base "$LENS_RELEASE_PROVIDER_API_BASE" \
'{model_name:$model,litellm_params:{model:$provider_model,api_base:$base,api_key:"os.environ/LENS_RELEASE_PROVIDER_API_KEY"}}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /model/new 200 register-provider-model "$qa_dir/request.json"
jq -e '.model_info.id | type == "string" and length > 0' "$qa_dir/release-response.json" > /dev/null
qualification_paid_calls=0
qualification_total_cost=0
fi
}
qualification_read_boundaries() {
if [[ "$qualification_mode" == smoke ]]; then return; fi
local phase=$1 tenant other credential own foreign
for tenant in a b; do
other=a
if [[ "$tenant" == a ]]; then other=b; fi
credential=$(jq -r .key "$qa_dir/tenant-$tenant.json")
own=$(cat "$qa_dir/tenant-$tenant-trace-id")
foreign=$(cat "$qa_dir/tenant-$other-trace-id")
qualification_request "$credential" GET "/v1/traces/$own" 200 "$phase-$tenant-own-trace"
jq -e --arg trace "$own" '.summary.trace_id == $trace' "$qa_dir/release-response.json" > /dev/null
qualification_request "$credential" GET "/v1/traces/$foreign" 404 "$phase-$tenant-other-team-denied"
qualification_request "$credential" GET "/v1/traces/$foreign?all_teams=1" 404 "$phase-$tenant-query-cannot-expand-scope"
jq -n --arg sql "SELECT TraceId AS trace_id FROM otel_traces WHERE TraceId IN ('$own','$foreign') ORDER BY TraceId" '{sql:$sql}' > "$qa_dir/request.json"
qualification_request "$credential" POST /v1/traces/query 200 "$phase-$tenant-scoped-sql" "$qa_dir/request.json"
jq -e --arg trace "$own" '[.data[].trace_id] == [$trace]' "$qa_dir/release-response.json" > /dev/null
qualification_request "$credential" GET /lens/signals 403 "$phase-$tenant-admin-surface-denied"
done
qualification_request "$(jq -r .key "$qa_dir/revoked-key.json")" GET /lens/service 401 "$phase-revocation-retained"
qualification_request invalid-release-key GET "/v1/traces/$own" 401 "$phase-invalid-key-denied"
qualification_record "$phase-auth-and-retained-data" passed
}
qualification_inference() {
if [[ "$qualification_mode" != release ]]; then return; fi
local phase=$1 credential response_id call_id cost attempt
test "$qualification_paid_calls" -lt 3
credential=$(jq -r .key "$qa_dir/tenant-a.json")
jq -n --arg model "$qualification_model" --arg marker "$(openssl rand -hex 16)" \
'{model:$model,messages:[{role:"user",content:("Reply with the single word ready. Test marker "+$marker)}],max_completion_tokens:64}' > "$qa_dir/request.json"
qualification_request "$credential" POST /v1/chat/completions 200 "$phase-real-inference" "$qa_dir/request.json"
qualification_paid_calls=$((qualification_paid_calls + 1))
cp "$qa_dir/release-response.json" "$qa_dir/completion.json"
jq -e '.choices[0].message.content | type == "string" and length > 0' "$qa_dir/completion.json" > /dev/null
jq -e '.usage.prompt_tokens > 0 and .usage.completion_tokens > 0 and .usage.total_tokens == (.usage.prompt_tokens + .usage.completion_tokens)' "$qa_dir/completion.json" > /dev/null
response_id=$(jq -er .id "$qa_dir/completion.json")
call_id=$(awk 'tolower($1)=="x-litellm-call-id:" {gsub("\r", "", $2); print $2}' "$qa_dir/release-response.headers")
cost=$(awk 'tolower($1)=="x-litellm-response-cost:" {gsub("\r", "", $2); print $2}' "$qa_dir/release-response.headers")
[[ "$call_id" =~ ^[A-Za-z0-9_.:-]+$ ]]
jq -en --arg cost "$cost" '($cost|tonumber) > 0' > /dev/null
for attempt in $(seq 1 60); do
qualification_request "$master_key" GET "/spend/logs?request_id=$call_id" 200 "$phase-billing-read"
if jq -e --arg id "$response_id" '[.[] | select(.request_id == $id)] | length == 1' "$qa_dir/release-response.json" > /dev/null; then break; fi
sleep 2
done
jq -e --arg id "$response_id" --arg cost "$cost" --arg team "$(jq -r .team "$qa_dir/tenant-a.json")" \
--slurpfile completion "$qa_dir/completion.json" \
'[.[] | select(.request_id == $id)] | length == 1 and (.[0] | .team_id == $team
and .prompt_tokens == $completion[0].usage.prompt_tokens
and .completion_tokens == $completion[0].usage.completion_tokens
and .total_tokens == $completion[0].usage.total_tokens
and ((.spend - ($cost|tonumber)) | fabs) < 0.00000001)' "$qa_dir/release-response.json" > /dev/null
qualification_total_cost=$(jq -en --arg total "$qualification_total_cost" --arg cost "$cost" '($total|tonumber)+($cost|tonumber)')
jq -en --arg total "$qualification_total_cost" '($total|tonumber) <= 0.10' > /dev/null
qualification_record "$phase-billing-matches-usage-and-team" passed
jq -nc --arg chart "$chart" --arg phase "$phase" --arg response_id "$response_id" \
--arg call_id "$call_id" --argjson cost "$cost" '{chart:$chart,case:($phase+"-paid-receipt"),result:"passed",response_id:$response_id,request_id:$call_id,cost_usd:$cost}' >> "$qualification_events"
}
qualification_outage() {
if [[ "$qualification_mode" == smoke ]]; then return; fi
local replicas selector
qualification_read_boundaries before-outage
qualification_inference before-outage
deployment_pods "$control" > "$qa_dir/outage-gateway-before.json"
replicas=$(kubectl -n "$namespace" get deployment lens-lens-worker -o jsonpath='{.spec.replicas}')
test "$replicas" = 1
selector=$(kubectl -n "$namespace" get deployment lens-lens-worker -o json | jq -r '.spec.selector.matchLabels|to_entries|map("\(.key)=\(.value)")|join(",")')
kubectl -n "$namespace" scale deployment/lens-lens-worker --replicas=0
kubectl -n "$namespace" wait --for=delete pods -l "$selector" --timeout=90s
qualification_request "$master_key" GET /lens/service 200 outage-service-status
jq -e '.configured == true and .connected == false' "$qa_dir/release-response.json" > /dev/null
qualification_request "$master_key" GET /lens/signals 503 outage-lens-unavailable
qualification_request invalid-release-key GET /lens/signals 401 outage-invalid-key-still-denied
qualification_inference during-outage
kubectl -n "$namespace" scale deployment/lens-lens-worker --replicas="$replicas"
kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s
forward_bundled
saved_trace
qualification_read_boundaries after-outage
qualification_inference after-outage
deployment_pods "$control" > "$qa_dir/outage-gateway-after.json"
cmp "$qa_dir/outage-gateway-before.json" "$qa_dir/outage-gateway-after.json"
qualification_record lens-outage-without-gateway-restart passed
}
qualification_bundled_postgres() {
local chart=bundled-postgresql release=lens-bootstrap
namespace=lens-bundled-postgresql
kubectl create namespace "$namespace"
master_key="sk-$(openssl rand -hex 24)"
printf '%s' "$master_key" > "$qa_dir/bootstrap-master"
kubectl -n "$namespace" create secret generic bootstrap-master --from-file="key=$qa_dir/bootstrap-master" > /dev/null
cat > "$qa_dir/bootstrap.yaml" <<YAML
fullnameOverride: $release
replicaCount: 0
image: {repository: lens-ci-monolith, tag: v0.0.0-lens-ci, pullPolicy: Never}
masterkeySecretName: bootstrap-master
masterkeySecretKey: key
envVars: {STORE_MODEL_IN_DB: "True"}
db: {deployStandalone: true, useExisting: false}
postgresql:
auth: {username: litellm, database: litellm, password: "$(openssl rand -hex 24)", postgresPassword: "$(openssl rand -hex 24)"}
primary: {persistence: {enabled: false}}
redis: {enabled: false}
lensWorker: {enabled: false}
migrationJob:
ttlSecondsAfterFinished: 3600
hooks: {helm: {enabled: false}, argocd: {enabled: false}}
proxy_config:
model_list: []
general_settings: {master_key: os.environ/PROXY_MASTER_KEY, store_model_in_db: true}
YAML
helm upgrade --install "$release" helm/litellm-helm -n "$namespace" -f "$qa_dir/bootstrap.yaml" \
--wait --wait-for-jobs --timeout 8m
kubectl -n "$namespace" get deployment "$release" -o json | jq -e '.spec.replicas == 0 and (.status.replicas // 0) == 0' > /dev/null
kubectl -n "$namespace" get job "$release-migrations" -o json \
| jq -e '.status.succeeded == 1 and .metadata.annotations["helm.sh/hook"] == null and .metadata.annotations["argocd.argoproj.io/hook"] == null' > /dev/null
qualification_record bundled-postgres-migrated-before-gateway-start passed
kubectl -n "$namespace" scale deployment/"$release" --replicas=1
kubectl -n "$namespace" rollout status deployment/"$release" --timeout=180s
forward "$release" 14418 4000
jq -n '{key_alias:"Bundled PostgreSQL persisted key"}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /key/generate 200 bundled-postgres-key-create "$qa_dir/request.json"
local generated
generated=$(jq -er .key "$qa_dir/release-response.json")
qualification_request "$generated" GET /v1/models 200 bundled-postgres-key-auth
kubectl -n "$namespace" rollout restart deployment/"$release"
kubectl -n "$namespace" rollout status deployment/"$release" --timeout=180s
stop_forwards
forward "$release" 14418 4000
qualification_request "$generated" GET /v1/models 200 bundled-postgres-key-survives-gateway-restart
jq -n --arg key "$generated" '{keys:[$key]}' > "$qa_dir/request.json"
qualification_request "$master_key" POST /key/delete 200 bundled-postgres-key-delete "$qa_dir/request.json"
qualification_request "$generated" GET /v1/models 401 bundled-postgres-key-revoked
qualification_record bundled-postgres-ordinary-job-two-phase-bootstrap passed
stop_forwards
kubectl delete namespace "$namespace" --wait=true
namespace=
}

View file

@ -1,60 +0,0 @@
from __future__ import annotations
import os
import subprocess
from pathlib import Path
from typing import Final
import pytest
HARNESS: Final = Path(__file__).resolve().parents[2] / "e2e/migrations/lens_release_qualification.sh"
@pytest.mark.parametrize("chart", ["litellm", "litellm-helm"])
def test_inference_reaches_gateway_and_management_reaches_backend(tmp_path: Path, chart: str) -> None:
curl: Final = tmp_path / "curl"
curl.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "${@: -1}" >> "$qa_dir/requests"\nprintf 200\n')
curl.chmod(0o755)
forward: Final = tmp_path / "forward"
forward.write_text('#!/usr/bin/env bash\nprintf "%s %s %s\\n" "$@" >> "$qa_dir/forwards"\n')
forward.chmod(0o755)
result: Final = subprocess.run(
[
"bash",
"-euo",
"pipefail",
"-c",
'source "$HARNESS"\n'
"qualification_forward_control\n"
"qualification_initialize\n"
"qualification_request fixture POST /v1/chat/completions 200 inference\n"
"qualification_request fixture GET /spend/logs 200 billing\n"
"qualification_request fixture POST /model/new 200 management\n",
],
env={
**os.environ,
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"HARNESS": str(HARNESS),
"qa_dir": str(tmp_path),
"chart": chart,
"qualification_mode": "release",
"control": "lens-backend" if chart == "litellm" else "lens",
"control_port": "4001" if chart == "litellm" else "4000",
"LENS_QUALIFICATION_RESULTS_DIR": "",
},
capture_output=True,
text=True,
timeout=10,
check=False,
)
assert result.returncode == 0, result.stderr
inference_port: Final = 14420 if chart == "litellm" else 14418
assert (tmp_path / "requests").read_text().splitlines() == [
f"http://127.0.0.1:{inference_port}/v1/chat/completions",
"http://127.0.0.1:14418/spend/logs",
"http://127.0.0.1:14418/model/new",
]
expected_forwards: Final = (
["lens-backend 14418 4001", "lens-gateway 14420 4000"] if chart == "litellm" else ["lens 14418 4000"]
)
assert (tmp_path / "forwards").read_text().splitlines() == expected_forwards