From f99d85e32136a4ce0422999d4487e73118825294 Mon Sep 17 00:00:00 2001 From: "ben.wangz" Date: Tue, 5 May 2026 15:43:42 +0800 Subject: [PATCH] [Fix] Helm: honor external DB secret in standalone mode --- .../ARGOCD_STANDALONE_DB_VALIDATION.md | 168 ++++++++++++++++++ .../litellm-helm/templates/_helpers.tpl | 29 +++ .../litellm-helm/templates/deployment.yaml | 8 +- .../templates/migrations-job.yaml | 16 +- .../templates/secret-dbcredentials.yaml | 4 +- 5 files changed, 218 insertions(+), 7 deletions(-) create mode 100644 deploy/charts/litellm-helm/ARGOCD_STANDALONE_DB_VALIDATION.md diff --git a/deploy/charts/litellm-helm/ARGOCD_STANDALONE_DB_VALIDATION.md b/deploy/charts/litellm-helm/ARGOCD_STANDALONE_DB_VALIDATION.md new file mode 100644 index 00000000000..3f2fd789635 --- /dev/null +++ b/deploy/charts/litellm-helm/ARGOCD_STANDALONE_DB_VALIDATION.md @@ -0,0 +1,168 @@ +# ArgoCD Validation Report: Standalone DB Secret Consistency + +This document records the validation process and results for the `litellm-helm` chart fix that unifies DB credential secret behavior in standalone DB mode. + +It is intended to be reused before opening a PR. + +## Scope + +- Chart: `deploy/charts/litellm-helm` +- Focus: standalone DB mode secret consistency +- Environment used for this run: + - Host: `47.118.19.219` + - Kubernetes: `k3s` + - GitOps: `ArgoCD` + - Chart distribution: in-cluster OCI registry + +## What Was Changed (Code) + +The following templates were updated: + +- `deploy/charts/litellm-helm/templates/_helpers.tpl` + - Added `litellm.dbCredentialsSecretName` helper with precedence: + 1. `db.dbCredentialsSecretName` + 2. `postgresql.auth.existingSecret` + 3. `-dbcredentials` fallback + - Added `litellm.shouldCreateDbCredentialsSecret` helper. + +- `deploy/charts/litellm-helm/templates/deployment.yaml` + - Standalone DB env secret refs now use `litellm.dbCredentialsSecretName`. + - `DATABASE_USERNAME`/`DATABASE_PASSWORD` keys are sourced from values with defaults. + +- `deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml` + - Fallback secret is created only when no external secret is configured. + +- `deploy/charts/litellm-helm/templates/migrations-job.yaml` + - Standalone mode secret usage aligned with deployment behavior. + +## ArgoCD Application YAML Used + +Validation was done with this file: + +- `/root/code/k8s-at-home/build/litellm.app.yaml` + +Key values used in that file: + +- `db.deployStandalone: true` +- `db.useExisting: false` +- `db.dbCredentialsSecretName: litellm-credentials` +- `postgresql.auth.existingSecret: litellm-credentials` +- `postgresql.auth.secretKeys.userPasswordKey: postgres-password` +- `postgresql.auth.secretKeys.adminPasswordKey: postgres-password` + +## Environment Setup Procedure + +1. Install `k3s` on host `47.118.19.219`. +2. Install ArgoCD (`v3.3.9`) in namespace `argocd`. +3. Create in-cluster registry namespace and service: + - namespace: `registry` + - service: `registry` NodePort (this run used `31666`) +4. Package chart: + - `helm package /root/code/litellm/deploy/charts/litellm-helm --destination /root/code/k8s-at-home/build` +5. Push package to registry from remote host: + - `helm push /root/litellm-helm-1.1.0.tgz oci://127.0.0.1:/helm-charts --plain-http` +6. Create runtime namespace/secrets: + - namespace: `agents` + - secret: `litellm-credentials` + - secret: `litellm-env-secret` +7. Apply ArgoCD Application: + - `kubectl apply -n argocd -f /root/litellm.app.yaml` + +## Operational Notes (Important) + +The following image/network behaviors were observed in this environment: + +- Some pulls from upstream registries are very slow or timeout. +- For stability, pre-pull and retag images using `m.daocloud.io` as needed. +- `local-path` helper pods require `rancher/mirrored-library-busybox:1.37.0`. + +Recommended prepull examples: + +```bash +k3s ctr -n k8s.io images pull m.daocloud.io/docker.io/rancher/mirrored-library-busybox:1.37.0 +k3s ctr -n k8s.io images tag m.daocloud.io/docker.io/rancher/mirrored-library-busybox:1.37.0 docker.io/rancher/mirrored-library-busybox:1.37.0 +``` + +For long pulls, run scripts in background and log to file (`nohup ... > logfile 2>&1 &`). + +## Secret Requirements for This Configuration + +When `deployment.yaml` reads DB credentials from `litellm-credentials`, ensure these keys exist: + +- `username` +- `password` +- `postgres-password` +- `redis-password` + +In this run, missing `username` caused: + +- Pod status: `CreateContainerConfigError` +- Event: `couldn't find key username in Secret agents/litellm-credentials` + +Fix: + +```bash +kubectl patch secret -n agents litellm-credentials --type merge -p '{"stringData":{"username":"litellm"}}' +kubectl delete pod -n agents -l app.kubernetes.io/name=litellm +``` + +## Validation Results + +After deployment stabilized: + +- `litellm-postgresql-0`: `Running` +- `litellm-redis-master-0`: `Running` +- `litellm`: `Running` + +Critical check for this fix: + +- `litellm-dbcredentials` was **not created** when external secret configuration was provided. + +Observed runtime behavior: + +- LiteLLM started successfully. +- Health endpoints returned `200` repeatedly. +- No Prisma `P1000` auth failure observed. + +## Clean Revalidation (Delete and Recreate) + +To verify the fix is stable and not dependent on old cluster state, we performed a full revalidation: + +1. Deleted ArgoCD application `litellm`. +2. Deleted namespace `agents`. +3. Repackaged chart locally. +4. Pushed rebuilt chart package again to in-cluster OCI registry. +5. Recreated `agents` namespace and required secrets. +6. Reapplied ArgoCD Application YAML and forced refresh. + +Observed results after redeploy: + +- `litellm-postgresql-0`: `Running` +- `litellm-redis-master-0`: `Running` +- `litellm`: `Running` +- `litellm-dbcredentials`: **not found** +- LiteLLM logs show normal startup and readiness checks. +- No `P1000` database authentication error. + +Note: ArgoCD status during this run appeared as `OutOfSync Healthy`; runtime workload health and secret behavior matched expected fix behavior. + +## Revalidation Checklist (Before PR) + +1. Re-run `helm template` checks for: + - deployment secret name resolution + - fallback secret creation gating +2. Re-deploy on clean environment with ArgoCD Application YAML. +3. Confirm no `litellm-dbcredentials` secret exists when external secret is configured. +4. Confirm `litellm`, `postgresql`, `redis` all reach `Running`. +5. Confirm LiteLLM logs do not include DB authentication errors. +6. Run functional API checks (`/v1/chat/completions` non-stream and stream) and capture outputs. + +## Suggested PR Attachment Contents + +- This validation document. +- The ArgoCD Application YAML used for reproduction. +- Command snippets used for: + - chart package/push + - app apply/sync + - key verification checks +- Final pod status and key log evidence. diff --git a/deploy/charts/litellm-helm/templates/_helpers.tpl b/deploy/charts/litellm-helm/templates/_helpers.tpl index 25b02dd5f37..6bebf2c44fa 100644 --- a/deploy/charts/litellm-helm/templates/_helpers.tpl +++ b/deploy/charts/litellm-helm/templates/_helpers.tpl @@ -96,3 +96,32 @@ Get redis service port {{ .Values.redis.master.service.ports.redis }} {{- end -}} {{- end -}} + +{{/* +Resolve DB credentials secret name for standalone DB mode. +Precedence: +1) db.dbCredentialsSecretName +2) postgresql.auth.existingSecret +3) -dbcredentials (legacy default) +*/}} +{{- define "litellm.dbCredentialsSecretName" -}} +{{- if .Values.db.dbCredentialsSecretName -}} +{{- .Values.db.dbCredentialsSecretName -}} +{{- else if .Values.postgresql.auth.existingSecret -}} +{{- .Values.postgresql.auth.existingSecret -}} +{{- else -}} +{{- printf "%s-dbcredentials" (include "litellm.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Whether chart should create the fallback -dbcredentials secret. +Only create it when no external secret is configured. +*/}} +{{- define "litellm.shouldCreateDbCredentialsSecret" -}} +{{- if and (not .Values.db.dbCredentialsSecretName) (not .Values.postgresql.auth.existingSecret) -}} +true +{{- else -}} +false +{{- end -}} +{{- end -}} diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 97123e5df69..81fb9b4039f 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -64,13 +64,13 @@ spec: - name: DATABASE_USERNAME valueFrom: secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: username + name: {{ include "litellm.dbCredentialsSecretName" . }} + key: {{ .Values.db.secret.usernameKey | default "username" }} - name: DATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: password + name: {{ include "litellm.dbCredentialsSecretName" . }} + key: {{ .Values.db.secret.passwordKey | default "password" }} - name: DATABASE_HOST value: {{ .Release.Name }}-postgresql - name: DATABASE_NAME diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index c3f32fe32f3..e7bf1c75eff 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -73,8 +73,22 @@ spec: - name: DATABASE_URL value: {{ .Values.db.url | quote }} {{- else if .Values.db.deployStandalone }} + - name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "litellm.dbCredentialsSecretName" . }} + key: {{ .Values.db.secret.usernameKey | default "username" }} + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "litellm.dbCredentialsSecretName" . }} + key: {{ .Values.db.secret.passwordKey | default "password" }} + - name: DATABASE_HOST + value: {{ .Release.Name }}-postgresql + - name: DATABASE_NAME + value: {{ .Values.postgresql.auth.database | default "litellm" }} - name: DATABASE_URL - value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} + value: {{ .Values.db.url | quote }} {{- end }} {{- if .Values.envVars }} {{- range $key, $val := .Values.envVars }} diff --git a/deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml b/deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml index 8851f5802f2..531f9bbb891 100644 --- a/deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml +++ b/deploy/charts/litellm-helm/templates/secret-dbcredentials.yaml @@ -1,4 +1,4 @@ -{{- if .Values.db.deployStandalone -}} +{{- if and .Values.db.deployStandalone (eq (include "litellm.shouldCreateDbCredentialsSecret" .) "true") -}} apiVersion: v1 kind: Secret metadata: @@ -9,4 +9,4 @@ data: username: {{ .Values.postgresql.auth.username | default "litellm" | b64enc }} password: {{ .Values.postgresql.auth.password | default "litellm" | b64enc }} type: Opaque -{{- end -}} \ No newline at end of file +{{- end -}}