test(lens): install and upgrade both Helm charts on Kubernetes

.github/workflows/lens-install-smoke.yml

@@ -0,0 +1,51 @@
+name: Lens installation smoke
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - helm/**
+      - deploy/lens/**
+      - litellm-rust/crates/lens/**
+      - tests/e2e/migrations/lens_helm_smoke.sh
+      - .github/workflows/lens-install-smoke.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: lens-install-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  helm-install:
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955
+        with:
+          persist-credentials: false
+      - name: Build the matching release images
+        run: |
+          for component in gateway backend ui migrations; do
+            docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci \
+              -f "$component/Dockerfile" -t "lens-ci-$component:v0.0.0-lens-ci" .
+          done
+          docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci \
+            -f Dockerfile -t lens-ci-monolith:v0.0.0-lens-ci .
+          docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci \
+            -f deploy/lens/Dockerfile -t lens-ci-worker:v0.0.0-lens-ci .
+      - name: Install pinned Kubernetes test tools
+        run: |
+          curl --fail --location --output "$RUNNER_TEMP/kind" \
+            https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
+          echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b  $RUNNER_TEMP/kind" | sha256sum --check
+          chmod +x "$RUNNER_TEMP/kind"
+          curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
+            https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
+          echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496  $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
+          tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
+          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+          echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
+      - name: Install, ingest, upgrade, and restart both charts
+        run: bash tests/e2e/migrations/lens_helm_smoke.sh

tests/e2e/migrations/lens_helm_smoke.sh

@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+qa_dir=$(mktemp -d)
+cluster=lens-install-ci
+forward_pids=()
+cleanup() {
+  for pid in "${forward_pids[@]}"; do kill "$pid" 2>/dev/null || true; done
+  kind delete cluster --name "$cluster"
+  rm -rf "$qa_dir"
+}
+trap cleanup EXIT
+umask 077
+export KUBECONFIG="$qa_dir/kubeconfig"
+kind create cluster --name "$cluster" \
+  --image kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f \
+  --wait 120s
+for component in gateway backend ui migrations monolith worker; do
+  kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci"
+done
+helm dependency build helm/litellm-helm
+
+api() {
+  curl --fail-with-body --silent --show-error --max-time 20 \
+    -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \
+    "http://127.0.0.1:14418$1" "${@:2}"
+}
+
+saved_trace() {
+  for attempt in $(seq 1 30); do
+    if api "/v1/traces/$trace_id" > "$qa_dir/saved.json" && \
+      jq -e --arg span "$span_id" 'any(.spans[]; .span_id == $span)' "$qa_dir/saved.json" > /dev/null; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+diagnose() {
+  kubectl -n "$namespace" get pods
+  kubectl -n "$namespace" get events --sort-by=.lastTimestamp | tail -30
+  kubectl -n "$namespace" logs --all-containers -l app.kubernetes.io/instance=lens --tail=50 || true
+  return 1
+}
+
+for chart in litellm-helm litellm; do
+  namespace="lens-$chart"
+  kubectl create namespace "$namespace"
+  master_key="sk-$(openssl rand -hex 24)"
+  kubectl -n "$namespace" create secret generic lens-secrets \
+    --from-literal="master-key=$master_key" \
+    --from-literal="service-token=$(openssl rand -hex 32)" \
+    --from-literal="url=http://clickhouse:8123" \
+    --from-literal=username=litellm --from-literal=password=isolated-helm-test
+  kubectl -n "$namespace" apply -f - <<'YAML'
+apiVersion: apps/v1
+kind: Deployment
+metadata: {name: postgres}
+spec:
+  selector: {matchLabels: {app: postgres}}
+  template:
+    metadata: {labels: {app: postgres}}
+    spec:
+      containers:
+        - name: postgres
+          image: postgres:16
+          env:
+            - {name: POSTGRES_DB, value: litellm}
+            - {name: POSTGRES_USER, value: litellm}
+            - {name: POSTGRES_PASSWORD, value: isolated-helm-test}
+          readinessProbe:
+            exec: {command: [pg_isready, -U, litellm, -d, litellm]}
+---
+apiVersion: v1
+kind: Service
+metadata: {name: postgres}
+spec:
+  selector: {app: postgres}
+  ports: [{port: 5432}]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata: {name: clickhouse}
+spec:
+  selector: {matchLabels: {app: clickhouse}}
+  template:
+    metadata: {labels: {app: clickhouse}}
+    spec:
+      containers:
+        - name: clickhouse
+          image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e
+          env: [{name: CLICKHOUSE_SKIP_USER_SETUP, value: "1"}]
+          readinessProbe:
+            httpGet: {path: /ping, port: 8123}
+---
+apiVersion: v1
+kind: Service
+metadata: {name: clickhouse}
+spec:
+  selector: {app: clickhouse}
+  ports: [{port: 8123}]
+YAML
+  kubectl -n "$namespace" rollout status deployment/postgres --timeout=180s
+  kubectl -n "$namespace" rollout status deployment/clickhouse --timeout=180s
+  cat > "$qa_dir/common.yaml" <<'YAML'
+fullnameOverride: lens
+lensWorker:
+  enabled: true
+  image: {repository: lens-ci-worker, tag: v0.0.0-lens-ci, pullPolicy: Never}
+  serviceTokenSecret: {name: lens-secrets, key: service-token}
+  clickhouseSecret: {name: lens-secrets, key: url}
+  clickhouseDatabase: existing_traces
+  retentionDays: 45
+  publicUrl: http://127.0.0.1:14419
+YAML
+  if [[ "$chart" == litellm-helm ]]; then
+    control=lens
+    control_port=4000
+    cat > "$qa_dir/chart.yaml" <<'YAML'
+image: {repository: lens-ci-monolith, tag: v0.0.0-lens-ci, pullPolicy: Never}
+masterkeySecretName: lens-secrets
+masterkeySecretKey: master-key
+envVars: {STORE_MODEL_IN_DB: "True"}
+db:
+  deployStandalone: false
+  useExisting: true
+  endpoint: postgres
+  secret: {name: lens-secrets, usernameKey: username, passwordKey: password}
+redis: {enabled: false}
+proxy_config:
+  model_list: []
+  general_settings:
+    master_key: os.environ/PROXY_MASTER_KEY
+    store_model_in_db: true
+    tracing: {enabled: true, store: {type: lens}}
+YAML
+  else
+    control=lens-backend
+    control_port=4001
+    cat > "$qa_dir/chart.yaml" <<'YAML'
+masterKey: {secretName: lens-secrets, secretKey: master-key}
+database:
+  writer:
+    host: postgres
+    dbname: litellm
+    passwordSecret: {name: lens-secrets, usernameKey: username, passwordKey: password}
+migrationJob:
+  image: {repository: lens-ci-migrations, tag: v0.0.0-lens-ci, pullPolicy: Never}
+gateway:
+  image: {repository: lens-ci-gateway, tag: v0.0.0-lens-ci, pullPolicy: Never}
+  numWorkers: 1
+  extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}]
+  hpa: {enabled: false}
+  resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}}
+  config:
+    create: true
+    proxy_config:
+      model_list: []
+      general_settings:
+        store_model_in_db: true
+        tracing: {enabled: true, store: {type: lens}}
+backend:
+  extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}]
+  image: {repository: lens-ci-backend, tag: v0.0.0-lens-ci, pullPolicy: Never}
+  hpa: {enabled: false}
+  resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}}
+ui:
+  image: {repository: lens-ci-ui, tag: v0.0.0-lens-ci, pullPolicy: Never}
+  hpa: {enabled: false}
+YAML
+  fi
+  install=(helm upgrade --install lens "helm/$chart" -n "$namespace" \
+    -f "$qa_dir/common.yaml" -f "$qa_dir/chart.yaml" --wait --timeout 8m)
+  "${install[@]}" || diagnose
+  kubectl -n "$namespace" port-forward "service/$control" "14418:$control_port" > "$qa_dir/control-forward.log" 2>&1 &
+  forward_pids+=("$!")
+  kubectl -n "$namespace" port-forward service/lens-lens-worker 14419:4318 > "$qa_dir/lens-forward.log" 2>&1 &
+  forward_pids+=("$!")
+  for attempt in $(seq 1 30); do
+    if api /lens/service > "$qa_dir/status.json" && jq -e '.connected and .status.storage_ready' "$qa_dir/status.json"; then break; fi
+    sleep 1
+  done
+  jq -e '.connected and .status.storage_ready' "$qa_dir/status.json"
+  api /lens/tracing/keys -d '{"name":"Helm smoke"}' > "$qa_dir/key.json"
+  tracing_key=$(jq -r .key "$qa_dir/key.json")
+  trace_id=$(openssl rand -hex 16)
+  span_id=$(openssl rand -hex 8)
+  jq -n --arg trace "$trace_id" --arg span "$span_id" --arg at "$(date +%s)000000000" \
+    '{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Helm trace",kind:1,
+      startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/trace.json"
+  curl --fail-with-body --silent --show-error --retry 10 --retry-all-errors --retry-delay 1 \
+    -H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \
+    -d "@$qa_dir/trace.json" http://127.0.0.1:14419/v1/traces
+  saved_trace
+  kubectl -n "$namespace" exec deployment/clickhouse -- clickhouse-client --query \
+    "SELECT count() FROM existing_traces.otel_traces WHERE TraceId = '$trace_id'" | rg -qx 1
+  "${install[@]}" || diagnose
+  saved_trace
+  for pid in "${forward_pids[@]}"; do kill "$pid"; wait "$pid" 2>/dev/null || true; done
+  forward_pids=()
+  kubectl -n "$namespace" rollout restart "deployment/$control" deployment/lens-lens-worker
+  kubectl -n "$namespace" rollout status "deployment/$control" --timeout=180s
+  kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s
+  kubectl -n "$namespace" port-forward "service/$control" "14418:$control_port" > "$qa_dir/control-forward.log" 2>&1 &
+  forward_pids+=("$!")
+  saved_trace
+  printf '%s: fresh install, direct ingestion, custom database, upgrade, and restart passed\n' "$chart"
+  for pid in "${forward_pids[@]}"; do kill "$pid"; wait "$pid" 2>/dev/null || true; done
+  forward_pids=()
+  kubectl delete namespace "$namespace" --wait=true
+done