From a06517c40b8efdbdaabda9c90c45cf94bba17abb Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 7 Jul 2026 11:51:41 -0700 Subject: [PATCH] fix(helm): require explicit metrics scrape credential instead of defaulting to the master key --- helm/litellm-helm/README.md | 3 +- .../templates/servicemonitor.yaml | 4 +- .../tests/servicemonitor_tests.yaml | 65 ------------------- helm/litellm-helm/values.yaml | 1 + 4 files changed, 6 insertions(+), 67 deletions(-) delete mode 100644 helm/litellm-helm/tests/servicemonitor_tests.yaml diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index 0b11e456b3e..70c88482ae4 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -54,9 +54,10 @@ If `db.useStackgresOperator` is used (not yet implemented): | `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | | `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | | `serviceMonitor.enabled` | Create a Prometheus Operator ServiceMonitor that scrapes the proxy's `/metrics/` endpoint | `false` | -| `serviceMonitor.authSecret.enabled` | Authenticate scrapes with a bearer token. The proxy requires authentication on `/metrics` unless `litellm_settings.require_auth_for_metrics_endpoint` is `false`. When `authSecret.name` is empty, the master key secret is used. The master key grants full admin access to the proxy, so consider pointing `authSecret.name`/`authSecret.key` at a secret holding a dedicated virtual key instead. | `true` | +| `serviceMonitor.authSecret.enabled` | Authenticate scrapes with a bearer token. The proxy requires authentication on `/metrics` unless `litellm_settings.require_auth_for_metrics_endpoint` is `false`. Requires either `authSecret.name`/`authSecret.key` or `authSecret.useMasterKey`. | `true` | | `serviceMonitor.authSecret.name` | Name of an existing Kubernetes Secret holding the bearer token used to scrape `/metrics/`. Must live in the same namespace as the ServiceMonitor (the release namespace). | `""` | | `serviceMonitor.authSecret.key` | Key within `serviceMonitor.authSecret.name` that holds the token. Required when `authSecret.name` is set. | `""` | +| `serviceMonitor.authSecret.useMasterKey` | Scrape with the master key secret (honors `masterkeySecretName`/`masterkeySecretKey`) when `authSecret.name` is not set. The master key grants full admin access to the proxy, so a dedicated virtual key via `authSecret.name`/`authSecret.key` is recommended. | `false` | #### Example `proxy_config` ConfigMap from values (default): diff --git a/helm/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml index 446ebf63f65..13655ae6670 100644 --- a/helm/litellm-helm/templates/servicemonitor.yaml +++ b/helm/litellm-helm/templates/servicemonitor.yaml @@ -38,9 +38,11 @@ spec: {{- if .authSecret.name }} name: {{ .authSecret.name }} key: {{ required "serviceMonitor.authSecret.key is required when serviceMonitor.authSecret.name is set" .authSecret.key }} - {{- else }} + {{- else if .authSecret.useMasterKey }} name: {{ $.Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" $)) }} key: {{ $.Values.masterkeySecretKey | default "masterkey" }} + {{- else }} + {{- fail "The proxy requires a bearer token to scrape /metrics. Set serviceMonitor.authSecret.name and serviceMonitor.authSecret.key to a secret holding a dedicated virtual key (recommended), or set serviceMonitor.authSecret.useMasterKey=true to scrape with the master key, or set serviceMonitor.authSecret.enabled=false if the proxy runs with require_auth_for_metrics_endpoint: false" }} {{- end }} {{- end }} {{- if .relabelings }} diff --git a/helm/litellm-helm/tests/servicemonitor_tests.yaml b/helm/litellm-helm/tests/servicemonitor_tests.yaml deleted file mode 100644 index f9487ba3a8b..00000000000 --- a/helm/litellm-helm/tests/servicemonitor_tests.yaml +++ /dev/null @@ -1,65 +0,0 @@ -suite: test service monitor -templates: - - servicemonitor.yaml -release: - name: my-release -tests: - - it: should not render a ServiceMonitor when disabled - set: - serviceMonitor.enabled: false - asserts: - - hasDocuments: - count: 0 - - it: should authenticate scrapes with the generated masterkey secret by default - set: - serviceMonitor.enabled: true - asserts: - - isKind: - of: ServiceMonitor - - equal: - path: spec.endpoints[0].authorization.type - value: Bearer - - equal: - path: spec.endpoints[0].authorization.credentials.name - value: my-release-litellm-masterkey - - equal: - path: spec.endpoints[0].authorization.credentials.key - value: masterkey - - it: should authenticate scrapes with masterkeySecretName and masterkeySecretKey when set - set: - serviceMonitor.enabled: true - masterkeySecretName: my-masterkey-secret - masterkeySecretKey: my-key - asserts: - - equal: - path: spec.endpoints[0].authorization.credentials.name - value: my-masterkey-secret - - equal: - path: spec.endpoints[0].authorization.credentials.key - value: my-key - - it: should authenticate scrapes with a custom auth secret when set - set: - serviceMonitor.enabled: true - serviceMonitor.authSecret.name: metrics-token - serviceMonitor.authSecret.key: token - asserts: - - equal: - path: spec.endpoints[0].authorization.credentials.name - value: metrics-token - - equal: - path: spec.endpoints[0].authorization.credentials.key - value: token - - it: should fail when a custom auth secret name is set without a key - set: - serviceMonitor.enabled: true - serviceMonitor.authSecret.name: metrics-token - asserts: - - failedTemplate: - errorMessage: serviceMonitor.authSecret.key is required when serviceMonitor.authSecret.name is set - - it: should not send credentials when authSecret is disabled - set: - serviceMonitor.enabled: true - serviceMonitor.authSecret.enabled: false - asserts: - - notExists: - path: spec.endpoints[0].authorization diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 51d7c13132a..975c393e32a 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -421,6 +421,7 @@ serviceMonitor: enabled: true name: "" key: "" + useMasterKey: false relabelings: [] # - targetLabel: __meta_kubernetes_pod_node_name # replacement: $1